From 48bd2e2be206e76db903f7ded79dbcfca31d80b5 Mon Sep 17 00:00:00 2001 From: Harry Hogg Date: Fri, 22 Oct 2021 14:57:02 +0100 Subject: [PATCH 1/6] chore(Snyk): Added workflow for outputting the Snyk report as JSON Signed-off-by: Harry Hogg Co-Authored-By: Himanshu Mishra Date: Mon, 25 Oct 2021 13:04:56 +0100 Subject: [PATCH 2/6] chore(Snyk): Added scripts for syncing Snyk vulnerabilities to Github issues. Signed-off-by: Harry Hogg --- package.json | 3 +- scripts/snyk-github-issue-sync.ts | 114 ++++++++++++++++++++++++++++++ yarn.lock | 78 ++++++++++++++++++-- 3 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 scripts/snyk-github-issue-sync.ts diff --git a/package.json b/package.json index 4aa6905085..4f25429569 100644 --- a/package.json +++ b/package.json @@ -53,15 +53,16 @@ }, "version": "1.0.0", "dependencies": { + "@octokit/rest": "^18.12.0", "@microsoft/api-documenter": "^7.13.47", "@microsoft/api-extractor": "^7.18.7", "@microsoft/api-extractor-model": "^7.13.5", "@microsoft/tsdoc": "^0.13.2" }, "devDependencies": { - "@types/webpack": "^5.28.0", "@changesets/cli": "^2.14.0", "@spotify/prettier-config": "^11.0.0", + "@types/webpack": "^5.28.0", "command-exists": "^1.2.9", "concurrently": "^6.0.0", "eslint-plugin-notice": "^0.9.10", diff --git a/scripts/snyk-github-issue-sync.ts b/scripts/snyk-github-issue-sync.ts new file mode 100644 index 0000000000..e20d7d4164 --- /dev/null +++ b/scripts/snyk-github-issue-sync.ts @@ -0,0 +1,114 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Octokit } from '@octokit/rest'; +import syncJsonOutput from '../snyk.json'; + +type Vulnerability = { + description: string; + id: string; + packages: Set; +}; + +const octokit = new Octokit({ + auth: process.env.GITHUB_TOKEN, +}); + +const fetchSnykGithubIssueMap = async (): Promise> => { + const snykGithubIssueMap: Record = {}; + + const iterator = octokit.paginate.iterator(octokit.rest.issues.listForRepo, { + owner: 'backstage', + repo: 'backstage', + per_page: 100, + labels: 'snyk', + }); + + for await (const { data: issues } of iterator) { + for (const issue of issues) { + const match = /\([([A-Z0-9-]+)\])/.exec(issue.title); + + if (match && match[1]) { + snykGithubIssueMap[match[1]] = issue.id; + } + } + } + + return snykGithubIssueMap; +}; + +const createGithubIssue = (vulnerability: Vulnerability) => { + console.log( + `Create issue for vulnerability ${ + vulnerability.id + } affecting packages ${Array.from(vulnerability.packages)}`, + ); + // TODO(hhogg): Create github issue with the contents from a Snyk issue. +}; + +const updateGithubIssue = ( + githubIssueId: number, + vulnerability: Vulnerability, +) => { + console.log( + `Update issue ${githubIssueId} for vulnerability ${vulnerability.id}`, + ); + // TODO(hhogg): Update github issue with the contents from a Snyk issue. +}; + +const closeGithubIssue = (githubIssueId: number) => { + console.log(`Delete issue ${githubIssueId}`); + // TODO(hhogg): Delete a github issue +}; + +(async () => { + const snykGithubIssueMap = await fetchSnykGithubIssueMap(); + const vulnerabilityStore: Record = {}; + + // Group the Snyk vulnerabilities, and aggregate the affecting packages. + syncJsonOutput.forEach(({ projectName, vulnerabilities }) => { + vulnerabilities.forEach( + ({ id, description }: { id: string; description: string }) => { + if (id !== undefined && description !== undefined) { + if (vulnerabilityStore[id]) { + vulnerabilityStore[id].packages.add(projectName); + } else { + vulnerabilityStore[id] = { + description, + id, + packages: new Set([projectName]), + }; + } + } + }, + ); + }); + + // Loop over the grouped vulnerabilities and create/update accordingly + Object.entries(vulnerabilityStore).forEach(([id, vulnerability]) => { + if (snykGithubIssueMap[id]) { + updateGithubIssue(snykGithubIssueMap[id], vulnerability); + } else { + createGithubIssue(vulnerability); + } + }); + + // Loop over the Github issues and delete accordingly. + Object.entries(snykGithubIssueMap).forEach(([snykId, githubIssueId]) => { + if (!snykGithubIssueMap[snykId]) { + closeGithubIssue(githubIssueId); + } + }); +})(); diff --git a/yarn.lock b/yarn.lock index c905fbea6f..92e2d56f9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4961,6 +4961,19 @@ before-after-hook "^2.1.0" universal-user-agent "^6.0.0" +"@octokit/core@^3.5.1": + version "3.5.1" + resolved "https://registry.npmjs.org/@octokit/core/-/core-3.5.1.tgz#8601ceeb1ec0e1b1b8217b960a413ed8e947809b" + integrity sha512-omncwpLVxMP+GLpLPgeGJBF6IWJFjXDS5flY5VbppePYX9XehevbDykRH9PdCdvqt9TS5AOTiDide7h0qrkHjw== + dependencies: + "@octokit/auth-token" "^2.4.4" + "@octokit/graphql" "^4.5.8" + "@octokit/request" "^5.6.0" + "@octokit/request-error" "^2.0.5" + "@octokit/types" "^6.0.3" + before-after-hook "^2.2.0" + universal-user-agent "^6.0.0" + "@octokit/endpoint@^6.0.1": version "6.0.3" resolved "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.3.tgz#dd09b599662d7e1b66374a177ab620d8cdf73487" @@ -4995,6 +5008,11 @@ "@octokit/types" "^6.12.2" btoa-lite "^1.0.0" +"@octokit/openapi-types@^11.2.0": + version "11.2.0" + resolved "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-11.2.0.tgz#b38d7fc3736d52a1e96b230c1ccd4a58a2f400a6" + integrity sha512-PBsVO+15KSlGmiI8QAzaqvsNlZlrDlyAJYcrXBCvVUxCp7VnXjkwPoFHgjEJXx3WF9BAwkA6nfCUA7i9sODzKA== + "@octokit/openapi-types@^7.3.2": version "7.3.2" resolved "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-7.3.2.tgz#065ce49b338043ec7f741316ce06afd4d459d944" @@ -5005,6 +5023,13 @@ resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" integrity sha512-93uGjlhUD+iNg1iWhUENAtJata6w5nE+V4urXOAlIXdco6xNZtUSfYY8dzp3Udy74aqO/B5UZL80x/YMa5PKRw== +"@octokit/plugin-paginate-rest@^2.16.8": + version "2.17.0" + resolved "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.17.0.tgz#32e9c7cab2a374421d3d0de239102287d791bce7" + integrity sha512-tzMbrbnam2Mt4AhuyCHvpRkS0oZ5MvwwcQPYGtMv4tUa5kkzG58SVB0fcsLulOZQeRnOgdkZWkRUiyBlh0Bkyw== + dependencies: + "@octokit/types" "^6.34.0" + "@octokit/plugin-paginate-rest@^2.6.2": version "2.7.0" resolved "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.7.0.tgz#6bb7b043c246e0654119a6ec4e72a172c9e2c7f3" @@ -5017,6 +5042,11 @@ resolved "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.2.tgz#394d59ec734cd2f122431fbaf05099861ece3c44" integrity sha512-oTJSNAmBqyDR41uSMunLQKMX0jmEXbwD1fpz8FG27lScV3RhtGfBa1/BBLym+PxcC16IBlF7KH9vP1BUYxA+Eg== +"@octokit/plugin-request-log@^1.0.4": + version "1.0.4" + resolved "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.4.tgz#5e50ed7083a613816b1e4a28aeec5fb7f1462e85" + integrity sha512-mLUsMkgP7K/cnFEw07kWqXGF5LKrOkD+lhCrKvPHXWDywAwuDUeDwWBpc69XK3pNX0uKiVt8g5z96PJ6z9xCFA== + "@octokit/plugin-rest-endpoint-methods@5.3.1": version "5.3.1" resolved "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.3.1.tgz#deddce769b4ec3179170709ab42e4e9e6195aaa9" @@ -5025,6 +5055,14 @@ "@octokit/types" "^6.16.2" deprecation "^2.3.1" +"@octokit/plugin-rest-endpoint-methods@^5.12.0": + version "5.13.0" + resolved "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-5.13.0.tgz#8c46109021a3412233f6f50d28786f8e552427ba" + integrity sha512-uJjMTkN1KaOIgNtUPMtIXDOjx6dGYysdIFhgA52x4xSadQCz3b/zJexvITDVpANnfKPW/+E0xkOvLntqMYpviA== + dependencies: + "@octokit/types" "^6.34.0" + deprecation "^2.3.1" + "@octokit/request-error@^2.0.0", "@octokit/request-error@^2.0.2", "@octokit/request-error@^2.0.5", "@octokit/request-error@^2.1.0": version "2.1.0" resolved "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.1.0.tgz#9e150357831bfc788d13a4fd4b1913d60c74d677" @@ -5056,6 +5094,16 @@ "@octokit/plugin-request-log" "^1.0.2" "@octokit/plugin-rest-endpoint-methods" "5.3.1" +"@octokit/rest@^18.12.0": + version "18.12.0" + resolved "https://registry.npmjs.org/@octokit/rest/-/rest-18.12.0.tgz#f06bc4952fc87130308d810ca9d00e79f6988881" + integrity sha512-gDPiOHlyGavxr72y0guQEhLsemgVjwRePayJ+FcKc2SJqKUbxbkvf5kAZEWA/MKvsfYlQAMVzNJE3ezQcxMJ2Q== + dependencies: + "@octokit/core" "^3.5.1" + "@octokit/plugin-paginate-rest" "^2.16.8" + "@octokit/plugin-request-log" "^1.0.4" + "@octokit/plugin-rest-endpoint-methods" "^5.12.0" + "@octokit/types@^5.0.0", "@octokit/types@^5.0.1": version "5.5.0" resolved "https://registry.npmjs.org/@octokit/types/-/types-5.5.0.tgz#e5f06e8db21246ca102aa28444cdb13ae17a139b" @@ -5070,6 +5118,13 @@ dependencies: "@octokit/openapi-types" "^7.3.2" +"@octokit/types@^6.34.0": + version "6.34.0" + resolved "https://registry.npmjs.org/@octokit/types/-/types-6.34.0.tgz#c6021333334d1ecfb5d370a8798162ddf1ae8218" + integrity sha512-s1zLBjWhdEI2zwaoSgyOFoKSl109CUcVBCc7biPJ3aAf6LGLU6szDvi31JPU7bxfla2lqfhjbbg/5DdFNxOwHw== + dependencies: + "@octokit/openapi-types" "^11.2.0" + "@octokit/webhooks-methods@^2.0.0": version "2.0.0" resolved "https://registry.npmjs.org/@octokit/webhooks-methods/-/webhooks-methods-2.0.0.tgz#1108b9ea661ca6c81e4a8bfa63a09eb27d5bc2db" @@ -7604,10 +7659,19 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@>=16.9.0", "@types/react@^16.9": - version "16.14.18" - resolved "https://registry.npmjs.org/@types/react/-/react-16.14.18.tgz#b2bcea05ee244fde92d409f91bd888ca8e54b20f" - integrity sha512-eeyqd1mqoG43mI0TvNKy9QNf1Tjz3DEOsRP3rlPo35OeMIt05I+v9RR8ZvL2GuYZeF2WAcLXJZMzu6zdz3VbtQ== +"@types/react@*", "@types/react@>=16.9.0": + version "17.0.33" + resolved "https://registry.npmjs.org/@types/react/-/react-17.0.33.tgz#e01ae3de7613dac1094569880bb3792732203ad5" + integrity sha512-pLWntxXpDPaU+RTAuSGWGSEL2FRTNyRQOjSWDke/rxRg14ncsZvx8AKWMWZqvc1UOaJIAoObdZhAWvRaHFi5rw== + dependencies: + "@types/prop-types" "*" + "@types/scheduler" "*" + csstype "^3.0.2" + +"@types/react@^16.9": + version "16.14.20" + resolved "https://registry.npmjs.org/@types/react/-/react-16.14.20.tgz#ff6e932ad71d92c27590e4a8667c7a53a7d0baad" + integrity sha512-SV7TaVc8e9E/5Xuv6TIyJ5VhQpZoVFJqX6IZgj5HZoFCtIDCArE3qXkcHlc6O/Ud4UwcMoX+tlvDA95YrKdLgA== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" @@ -9778,6 +9842,11 @@ before-after-hook@^2.1.0: resolved "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz#b6c03487f44e24200dd30ca5e6a1979c5d2fb635" integrity sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A== +before-after-hook@^2.2.0: + version "2.2.2" + resolved "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.2.tgz#a6e8ca41028d90ee2c24222f201c90956091613e" + integrity sha512-3pZEU3NT5BFUo/AD5ERPWOgQOCZITni6iavr5AUw5AUwQjMlI0kzu5btnyD39AF0gUEsDPwJT+oY1ORBJijPjQ== + better-opn@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/better-opn/-/better-opn-2.1.1.tgz#94a55b4695dc79288f31d7d0e5f658320759f7c6" @@ -20351,7 +20420,6 @@ minipass-fetch@^1.3.0, minipass-fetch@^1.3.2: resolved "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.3.3.tgz#34c7cea038c817a8658461bf35174551dce17a0a" integrity sha512-akCrLDWfbdAWkMLBxJEeWTdNsjML+dt5YgOI4gJ53vuO0vrmYQkUPxa6j6V65s9CcePIr2SSWqjT2EcrNseryQ== dependencies: - encoding "^0.1.12" minipass "^3.1.0" minipass-sized "^1.0.3" minizlib "^2.0.0" From 3cb1e8e6ee5f7561905c639b98427fc9bd109e71 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 26 Oct 2021 14:16:29 +0200 Subject: [PATCH 3/6] fix types/react version Co-authored-by: Harry Hogg Signed-off-by: Himanshu Mishra --- package.json | 2 +- scripts/snyk-github-issue-sync.ts | 2 ++ yarn.lock | 17 ++++------------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 4f25429569..77ec43ee24 100644 --- a/package.json +++ b/package.json @@ -53,13 +53,13 @@ }, "version": "1.0.0", "dependencies": { - "@octokit/rest": "^18.12.0", "@microsoft/api-documenter": "^7.13.47", "@microsoft/api-extractor": "^7.18.7", "@microsoft/api-extractor-model": "^7.13.5", "@microsoft/tsdoc": "^0.13.2" }, "devDependencies": { + "@octokit/rest": "^18.12.0", "@changesets/cli": "^2.14.0", "@spotify/prettier-config": "^11.0.0", "@types/webpack": "^5.28.0", diff --git a/scripts/snyk-github-issue-sync.ts b/scripts/snyk-github-issue-sync.ts index e20d7d4164..52e39da0cd 100644 --- a/scripts/snyk-github-issue-sync.ts +++ b/scripts/snyk-github-issue-sync.ts @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +// eslint-disable-next-line import/no-extraneous-dependencies import { Octokit } from '@octokit/rest'; +// The GitHub workflow .github/workflows/ import syncJsonOutput from '../snyk.json'; type Vulnerability = { diff --git a/yarn.lock b/yarn.lock index 92e2d56f9d..71b2fb89d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7659,19 +7659,10 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@>=16.9.0": - version "17.0.33" - resolved "https://registry.npmjs.org/@types/react/-/react-17.0.33.tgz#e01ae3de7613dac1094569880bb3792732203ad5" - integrity sha512-pLWntxXpDPaU+RTAuSGWGSEL2FRTNyRQOjSWDke/rxRg14ncsZvx8AKWMWZqvc1UOaJIAoObdZhAWvRaHFi5rw== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "*" - csstype "^3.0.2" - -"@types/react@^16.9": - version "16.14.20" - resolved "https://registry.npmjs.org/@types/react/-/react-16.14.20.tgz#ff6e932ad71d92c27590e4a8667c7a53a7d0baad" - integrity sha512-SV7TaVc8e9E/5Xuv6TIyJ5VhQpZoVFJqX6IZgj5HZoFCtIDCArE3qXkcHlc6O/Ud4UwcMoX+tlvDA95YrKdLgA== +"@types/react@*", "@types/react@>=16.9.0", "@types/react@^16.9": + version "16.14.18" + resolved "https://registry.npmjs.org/@types/react/-/react-16.14.18.tgz#b2bcea05ee244fde92d409f91bd888ca8e54b20f" + integrity sha512-eeyqd1mqoG43mI0TvNKy9QNf1Tjz3DEOsRP3rlPo35OeMIt05I+v9RR8ZvL2GuYZeF2WAcLXJZMzu6zdz3VbtQ== dependencies: "@types/prop-types" "*" "@types/scheduler" "*" From bf76bb7a1d88762307353dc623de86ab84eb1a34 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 26 Oct 2021 14:43:30 +0200 Subject: [PATCH 4/6] create github issue with a formatted body Co-authored-by: Harry Hogg Signed-off-by: Himanshu Mishra --- scripts/snyk-github-issue-sync.ts | 58 ++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/snyk-github-issue-sync.ts b/scripts/snyk-github-issue-sync.ts index 52e39da0cd..ed5082717d 100644 --- a/scripts/snyk-github-issue-sync.ts +++ b/scripts/snyk-github-issue-sync.ts @@ -15,15 +15,22 @@ */ // eslint-disable-next-line import/no-extraneous-dependencies import { Octokit } from '@octokit/rest'; -// The GitHub workflow .github/workflows/ -import syncJsonOutput from '../snyk.json'; +// Generated by GitHub workflow .github/workflows/snyk-github-issue-creator +import synkJsonOutput from '../snyk.json'; + +// Pattern for a GitHub Issue title +// Snyk vulnerability [Vulnerability ID] type Vulnerability = { description: string; - id: string; + snykId: string; packages: Set; }; +// Remember to fix me! +const GH_OWNER = 'orkohunter'; +const GH_REPO = 'backstage'; + const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, }); @@ -32,14 +39,16 @@ const fetchSnykGithubIssueMap = async (): Promise> => { const snykGithubIssueMap: Record = {}; const iterator = octokit.paginate.iterator(octokit.rest.issues.listForRepo, { - owner: 'backstage', - repo: 'backstage', + // TODO(Harry/Himanshu): Use a CLI flag for these values. + owner: GH_OWNER, + repo: GH_REPO, per_page: 100, - labels: 'snyk', + labels: 'snyk-vulnerability', }); for await (const { data: issues } of iterator) { for (const issue of issues) { + // Gets the Vulnerability ID from square braces const match = /\([([A-Z0-9-]+)\])/.exec(issue.title); if (match && match[1]) { @@ -51,13 +60,31 @@ const fetchSnykGithubIssueMap = async (): Promise> => { return snykGithubIssueMap; }; +const generateIssueBody = (vulnerability: Vulnerability) => { + let issueBody = ''; + issueBody += '## Affecting Packages/Plugins\n'; + vulnerability.packages.forEach(pkgName => { + issueBody += `* ${pkgName}\n`; + }); + // TODO: Use displayTargetFile in snyk.json to create hyperlinks + issueBody += '\n'; + issueBody += vulnerability.description; + return issueBody; +}; + const createGithubIssue = (vulnerability: Vulnerability) => { console.log( `Create issue for vulnerability ${ - vulnerability.id + vulnerability.snykId } affecting packages ${Array.from(vulnerability.packages)}`, ); - // TODO(hhogg): Create github issue with the contents from a Snyk issue. + octokit.issues.create({ + owner: GH_OWNER, + repo: GH_REPO, + title: `Snyk vulnerability [${vulnerability.snykId}]`, + labels: ['snyk-vulnerability', 'help wanted'], + body: generateIssueBody(vulnerability), + }); }; const updateGithubIssue = ( @@ -65,7 +92,7 @@ const updateGithubIssue = ( vulnerability: Vulnerability, ) => { console.log( - `Update issue ${githubIssueId} for vulnerability ${vulnerability.id}`, + `Update issue ${githubIssueId} for vulnerability ${vulnerability.snykId}`, ); // TODO(hhogg): Update github issue with the contents from a Snyk issue. }; @@ -75,12 +102,12 @@ const closeGithubIssue = (githubIssueId: number) => { // TODO(hhogg): Delete a github issue }; -(async () => { +async function main() { const snykGithubIssueMap = await fetchSnykGithubIssueMap(); const vulnerabilityStore: Record = {}; // Group the Snyk vulnerabilities, and aggregate the affecting packages. - syncJsonOutput.forEach(({ projectName, vulnerabilities }) => { + synkJsonOutput.forEach(({ projectName, vulnerabilities }) => { vulnerabilities.forEach( ({ id, description }: { id: string; description: string }) => { if (id !== undefined && description !== undefined) { @@ -89,7 +116,7 @@ const closeGithubIssue = (githubIssueId: number) => { } else { vulnerabilityStore[id] = { description, - id, + snykId: id, packages: new Set([projectName]), }; } @@ -113,4 +140,9 @@ const closeGithubIssue = (githubIssueId: number) => { closeGithubIssue(githubIssueId); } }); -})(); +} + +main().catch(error => { + console.error(error.stack); + process.exit(1); +}); From 7205d37a142e5814518feee3cfb7046342e194f3 Mon Sep 17 00:00:00 2001 From: Harry Hogg Date: Thu, 28 Oct 2021 13:04:14 +0100 Subject: [PATCH 5/6] Updated script to create, update and close github issues Signed-off-by: Harry Hogg --- .../workflows/snyk-github-issue-creator.yml | 27 --- .github/workflows/snyk-monitor.yml | 5 + package.json | 3 +- scripts/snyk-github-issue-sync.ts | 182 +++++++++++------- 4 files changed, 124 insertions(+), 93 deletions(-) delete mode 100644 .github/workflows/snyk-github-issue-creator.yml diff --git a/.github/workflows/snyk-github-issue-creator.yml b/.github/workflows/snyk-github-issue-creator.yml deleted file mode 100644 index ad50eef0cc..0000000000 --- a/.github/workflows/snyk-github-issue-creator.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Create and Update Github Issues from Snyk report - -on: - [push, pull_request] - # workflow_dispatch: - # pull_request: - # schedule: - # - cron: '0 */4 * * *' # every 4 hours - -jobs: - sync: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Run Snyk to check for vulnerabilities - uses: snyk/actions/node@master - continue-on-error: - env: - SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - with: - args: --yarn-workspaces --strict-out-of-sync=false - json: true - - - name: Run the Snyk Github Issue command - run: yarn ts-node scripts/snyk-github-issue-sync.ts diff --git a/.github/workflows/snyk-monitor.yml b/.github/workflows/snyk-monitor.yml index 0adb6c5bcb..32df37f35b 100644 --- a/.github/workflows/snyk-monitor.yml +++ b/.github/workflows/snyk-monitor.yml @@ -43,9 +43,14 @@ jobs: --org=backstage-dgh --strict-out-of-sync=false --sarif-file-output=snyk.sarif + --json-file-output=snyk.json + json: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - name: Upload Snyk report uses: github/codeql-action/upload-sarif@v1 with: sarif_file: snyk.sarif + + - name: Update Github issues + run: yarn ts-node scripts/snyk-github-issue-sync.ts diff --git a/package.json b/package.json index 77ec43ee24..94de50e792 100644 --- a/package.json +++ b/package.json @@ -59,8 +59,8 @@ "@microsoft/tsdoc": "^0.13.2" }, "devDependencies": { - "@octokit/rest": "^18.12.0", "@changesets/cli": "^2.14.0", + "@octokit/rest": "^18.12.0", "@spotify/prettier-config": "^11.0.0", "@types/webpack": "^5.28.0", "command-exists": "^1.2.9", @@ -70,6 +70,7 @@ "husky": "^6.0.0", "lerna": "^4.0.0", "lint-staged": "^11.1.2", + "minimist": "^1.2.5", "prettier": "^2.2.1", "shx": "^0.3.2", "yarn-lock-check": "^1.0.5" diff --git a/scripts/snyk-github-issue-sync.ts b/scripts/snyk-github-issue-sync.ts index ed5082717d..d7f64cf1ff 100644 --- a/scripts/snyk-github-issue-sync.ts +++ b/scripts/snyk-github-issue-sync.ts @@ -13,46 +13,65 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// eslint-disable-next-line import/no-extraneous-dependencies +/* eslint-disable import/no-extraneous-dependencies */ import { Octokit } from '@octokit/rest'; +import minimist from 'minimist'; // Generated by GitHub workflow .github/workflows/snyk-github-issue-creator import synkJsonOutput from '../snyk.json'; -// Pattern for a GitHub Issue title -// Snyk vulnerability [Vulnerability ID] - type Vulnerability = { description: string; + packages: { + name: string; + target: string; + }[]; snykId: string; - packages: Set; }; -// Remember to fix me! -const GH_OWNER = 'orkohunter'; +const argv = minimist(process.argv.slice(2)); + +const GH_OWNER = 'backstage'; const GH_REPO = 'backstage'; +const SNYK_GH_LABEL = 'snyk-vulnerability'; +const SNYK_ID_REGEX = /\[([A-Z0-9-:]+)]/i; + +const isDryRun = 'dryrun' in argv; + +if (!process.env.GITHUB_TOKEN) { + console.error('GITHUB_TOKEN is not set. Please provide a Github token'); + process.exit(1); +} const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN, }); +if (isDryRun) { + console.log( + '⚠️ Running in dryrun mode, no issues will be updated on Github ⚠️', + ); +} + const fetchSnykGithubIssueMap = async (): Promise> => { const snykGithubIssueMap: Record = {}; const iterator = octokit.paginate.iterator(octokit.rest.issues.listForRepo, { - // TODO(Harry/Himanshu): Use a CLI flag for these values. owner: GH_OWNER, repo: GH_REPO, per_page: 100, - labels: 'snyk-vulnerability', + state: 'open', + labels: SNYK_GH_LABEL, }); for await (const { data: issues } of iterator) { for (const issue of issues) { // Gets the Vulnerability ID from square braces - const match = /\([([A-Z0-9-]+)\])/.exec(issue.title); + const match = SNYK_ID_REGEX.exec(issue.title); if (match && match[1]) { - snykGithubIssueMap[match[1]] = issue.id; + snykGithubIssueMap[match[1]] = issue.number; + } else { + console.log(`Unmatched Snyk ID for ${issue.title}`); } } } @@ -60,86 +79,119 @@ const fetchSnykGithubIssueMap = async (): Promise> => { return snykGithubIssueMap; }; -const generateIssueBody = (vulnerability: Vulnerability) => { - let issueBody = ''; - issueBody += '## Affecting Packages/Plugins\n'; - vulnerability.packages.forEach(pkgName => { - issueBody += `* ${pkgName}\n`; - }); - // TODO: Use displayTargetFile in snyk.json to create hyperlinks - issueBody += '\n'; - issueBody += vulnerability.description; - return issueBody; -}; +const generateIssueBody = (vulnerability: Vulnerability) => ` +## Affecting Packages/Plugins -const createGithubIssue = (vulnerability: Vulnerability) => { +${Array.from(vulnerability.packages).map( + ({ name, target }) => `* [${name}](${target})\n`, +)} + +${vulnerability.description} +`; + +const createGithubIssue = async (vulnerability: Vulnerability) => { console.log( - `Create issue for vulnerability ${ - vulnerability.snykId - } affecting packages ${Array.from(vulnerability.packages)}`, + `Create Github Issue for Snyk Vulnerability ${vulnerability.snykId}`, ); - octokit.issues.create({ - owner: GH_OWNER, - repo: GH_REPO, - title: `Snyk vulnerability [${vulnerability.snykId}]`, - labels: ['snyk-vulnerability', 'help wanted'], - body: generateIssueBody(vulnerability), + + vulnerability.packages.forEach(({ name, target }) => { + console.log(`- ${name} [${target}]`); }); + + if (!isDryRun) { + await octokit.issues.create({ + owner: GH_OWNER, + repo: GH_REPO, + title: `Snyk vulnerability [${vulnerability.snykId}]`, + labels: [SNYK_GH_LABEL, 'help wanted'], + body: generateIssueBody(vulnerability), + }); + } }; -const updateGithubIssue = ( +const updateGithubIssue = async ( githubIssueId: number, vulnerability: Vulnerability, ) => { console.log( - `Update issue ${githubIssueId} for vulnerability ${vulnerability.snykId}`, + `Update Github Issue #${githubIssueId} for Snky Vulnerability ${vulnerability.snykId}`, ); - // TODO(hhogg): Update github issue with the contents from a Snyk issue. + + if (!isDryRun) { + await octokit.issues.update({ + owner: GH_OWNER, + repo: GH_REPO, + issue_number: githubIssueId, + body: generateIssueBody(vulnerability), + }); + } }; -const closeGithubIssue = (githubIssueId: number) => { - console.log(`Delete issue ${githubIssueId}`); - // TODO(hhogg): Delete a github issue +const closeGithubIssue = async (githubIssueId: number) => { + console.log(`Closing Github Issue #${githubIssueId}`); + + if (!isDryRun) { + await octokit.issues.update({ + owner: GH_OWNER, + repo: GH_REPO, + issue_number: githubIssueId, + state: 'closed', + }); + } }; async function main() { const snykGithubIssueMap = await fetchSnykGithubIssueMap(); const vulnerabilityStore: Record = {}; - // Group the Snyk vulnerabilities, and aggregate the affecting packages. - synkJsonOutput.forEach(({ projectName, vulnerabilities }) => { - vulnerabilities.forEach( - ({ id, description }: { id: string; description: string }) => { - if (id !== undefined && description !== undefined) { - if (vulnerabilityStore[id]) { - vulnerabilityStore[id].packages.add(projectName); - } else { - vulnerabilityStore[id] = { - description, - snykId: id, - packages: new Set([projectName]), - }; + // Group the Snyk vulnerabilities, and link back to the affecting packages. + synkJsonOutput.forEach( + ({ projectName, displayTargetFile, vulnerabilities }) => { + vulnerabilities.forEach( + ({ id, description }: { id: string; description: string }) => { + if (id !== undefined && description !== undefined) { + if (vulnerabilityStore[id]) { + if ( + !vulnerabilityStore[id].packages.some( + ({ name }) => name === projectName, + ) + ) { + vulnerabilityStore[id].packages.push({ + name: projectName, + target: displayTargetFile, + }); + } + } else { + vulnerabilityStore[id] = { + description, + snykId: id, + packages: [ + { + name: projectName, + target: displayTargetFile, + }, + ], + }; + } } - } - }, - ); - }); + }, + ); + }, + ); - // Loop over the grouped vulnerabilities and create/update accordingly - Object.entries(vulnerabilityStore).forEach(([id, vulnerability]) => { + for (const [id, vulnerability] of Object.entries(vulnerabilityStore)) { if (snykGithubIssueMap[id]) { - updateGithubIssue(snykGithubIssueMap[id], vulnerability); + await updateGithubIssue(snykGithubIssueMap[id], vulnerability); } else { - createGithubIssue(vulnerability); + await createGithubIssue(vulnerability); } - }); + } - // Loop over the Github issues and delete accordingly. - Object.entries(snykGithubIssueMap).forEach(([snykId, githubIssueId]) => { - if (!snykGithubIssueMap[snykId]) { - closeGithubIssue(githubIssueId); + for (const [snykId, githubIssueId] of Object.entries(snykGithubIssueMap)) { + if (!vulnerabilityStore[snykId]) { + await closeGithubIssue(githubIssueId); } - }); + } } main().catch(error => { From bd74a61eaabca24e8cdcedefe70beb21e6bff135 Mon Sep 17 00:00:00 2001 From: Harry Hogg Date: Mon, 8 Nov 2021 11:03:58 +0000 Subject: [PATCH 6/6] Updated to run on a separate workflow every 4 hours. Signed-off-by: Harry Hogg --- .github/workflows/snyk-github-issue-sync.yml | 23 ++++++++++++++++++++ .github/workflows/snyk-monitor.yml | 5 ----- 2 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/snyk-github-issue-sync.yml diff --git a/.github/workflows/snyk-github-issue-sync.yml b/.github/workflows/snyk-github-issue-sync.yml new file mode 100644 index 0000000000..34eb5c1ca4 --- /dev/null +++ b/.github/workflows/snyk-github-issue-sync.yml @@ -0,0 +1,23 @@ +name: 'Snyk Github Issue Sync' + +on: + schedule: + - cron: '0 */4 * * *' + +jobs: + sync: + steps: + - uses: actions/checkout@v2 + - name: Create Snyk report + uses: snyk/actions/node@master + with: + args: > + --yarn-workspaces + --org=backstage-dgh + --strict-out-of-sync=false + --json-file-output=snyk.json + json: true + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + - name: Update Github issues + run: yarn ts-node scripts/snyk-github-issue-sync.ts diff --git a/.github/workflows/snyk-monitor.yml b/.github/workflows/snyk-monitor.yml index 32df37f35b..0adb6c5bcb 100644 --- a/.github/workflows/snyk-monitor.yml +++ b/.github/workflows/snyk-monitor.yml @@ -43,14 +43,9 @@ jobs: --org=backstage-dgh --strict-out-of-sync=false --sarif-file-output=snyk.sarif - --json-file-output=snyk.json - json: true env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - name: Upload Snyk report uses: github/codeql-action/upload-sarif@v1 with: sarif_file: snyk.sarif - - - name: Update Github issues - run: yarn ts-node scripts/snyk-github-issue-sync.ts