From 34bb9d171432cc85de97929bf7bf9f60aa447c68 Mon Sep 17 00:00:00 2001 From: blam Date: Sat, 28 Nov 2020 22:11:56 +0100 Subject: [PATCH 01/74] feat: add thugboat --- .tugboat/config.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .tugboat/config.yml diff --git a/.tugboat/config.yml b/.tugboat/config.yml new file mode 100644 index 0000000000..ef037c0d28 --- /dev/null +++ b/.tugboat/config.yml @@ -0,0 +1,4 @@ +services: + backstage: + build: + dockerfile: ./packages/backend/Dockerfile From dbea1562dcd808d4350e66deb37f64ea7f47133d Mon Sep 17 00:00:00 2001 From: blam Date: Sat, 28 Nov 2020 22:21:56 +0100 Subject: [PATCH 02/74] chore: rework tugboat build --- .tugboat/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index ef037c0d28..612cbdb5af 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -1,4 +1,9 @@ services: backstage: - build: - dockerfile: ./packages/backend/Dockerfile + image: tugboatqa/node:lts + commands: + init: + - yarn + - yarn build + start: + - yarn start-backend From b303ee9fb170cc639efdc3b883263c3147ea4fc3 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 13:11:40 +0100 Subject: [PATCH 03/74] chore: updating tugboat config to build and tsc --- .tugboat/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 612cbdb5af..14cfd6dc85 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -3,7 +3,8 @@ services: image: tugboatqa/node:lts commands: init: - - yarn + - yarn install + - yarn tsc - yarn build start: - yarn start-backend From 4ee32a78f140f69f68d875cd610d74e0d31202f0 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 1 Dec 2020 20:08:47 +0100 Subject: [PATCH 04/74] chore: set as default --- .tugboat/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 14cfd6dc85..b3195d6254 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -1,10 +1,11 @@ services: backstage: image: tugboatqa/node:lts + default: true commands: init: - yarn install - yarn tsc - yarn build start: - - yarn start-backend + - APP_CONFIG_backend_listen_port=80 yarn start-backend From e0f22774801788fd2584879c19eb2dfd923ebf46 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 2 Dec 2020 10:24:59 +0100 Subject: [PATCH 05/74] chore: fixing container port --- .tugboat/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index b3195d6254..ef696e28de 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -1,6 +1,7 @@ services: backstage: image: tugboatqa/node:lts + expose: 7000 default: true commands: init: @@ -8,4 +9,4 @@ services: - yarn tsc - yarn build start: - - APP_CONFIG_backend_listen_port=80 yarn start-backend + - yarn start-backend From 8caf5cb0eacc980ace3e295c94a35c9df4b0e12f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Dec 2020 00:15:30 +0100 Subject: [PATCH 06/74] chore: make the start-backend command backgrounded --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index ef696e28de..3bc8865493 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -9,4 +9,4 @@ services: - yarn tsc - yarn build start: - - yarn start-backend + - yarn start-backend & From 83006ba83ab31387bb0e10055f756fdfee790f6a Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 3 Dec 2020 00:17:51 +0100 Subject: [PATCH 07/74] chore: fixing some base build preview things --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 3bc8865493..20f716785c 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -4,7 +4,7 @@ services: expose: 7000 default: true commands: - init: + build: - yarn install - yarn tsc - yarn build From 6800da78d9237da704105d9d49797c51b31be4c6 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 21 Jan 2021 22:26:34 +0100 Subject: [PATCH 08/74] integration: Fix default branch API url for hosted bitbucket server --- .changeset/eleven-lamps-hide.md | 5 +++++ packages/integration/src/bitbucket/core.test.ts | 6 +++--- packages/integration/src/bitbucket/core.ts | 8 ++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 .changeset/eleven-lamps-hide.md diff --git a/.changeset/eleven-lamps-hide.md b/.changeset/eleven-lamps-hide.md new file mode 100644 index 0000000000..809de25c82 --- /dev/null +++ b/.changeset/eleven-lamps-hide.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': patch +--- + +Fix default branch API url for custom hosted Bitbucket server diff --git a/packages/integration/src/bitbucket/core.test.ts b/packages/integration/src/bitbucket/core.test.ts index 39707976fe..1287ab4d66 100644 --- a/packages/integration/src/bitbucket/core.test.ts +++ b/packages/integration/src/bitbucket/core.test.ts @@ -116,7 +116,7 @@ describe('bitbucket core', () => { }; worker.use( rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default', + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/default-branch', (_, res, ctx) => res( ctx.status(200), @@ -144,7 +144,7 @@ describe('bitbucket core', () => { }; worker.use( rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default', + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/default-branch', (_, res, ctx) => res( ctx.status(200), @@ -231,7 +231,7 @@ describe('bitbucket core', () => { }; worker.use( rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/branches/default', + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/default-branch', (_, res, ctx) => res( ctx.status(200), diff --git a/packages/integration/src/bitbucket/core.ts b/packages/integration/src/bitbucket/core.ts index ae61df497d..f5235d8189 100644 --- a/packages/integration/src/bitbucket/core.ts +++ b/packages/integration/src/bitbucket/core.ts @@ -31,9 +31,10 @@ export async function getBitbucketDefaultBranch( const { name: repoName, owner: project, resource } = parseGitUrl(url); const isHosted = resource === 'bitbucket.org'; + // Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp184 const branchUrl = isHosted ? `${config.apiBaseUrl}/repositories/${project}/${repoName}` - : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/branches/default`; + : `${config.apiBaseUrl}/projects/${project}/repos/${repoName}/default-branch`; const response = await fetch(branchUrl, getBitbucketRequestOptions(config)); if (!response.ok) { @@ -50,7 +51,10 @@ export async function getBitbucketDefaultBranch( defaultBranch = displayId; } if (!defaultBranch) { - throw new Error(`Failed to read default branch from ${branchUrl}`); + throw new Error( + `Failed to read default branch from ${branchUrl}. ` + + `Response ${response.status} ${response.json()}`, + ); } return defaultBranch; } From fac91bcc5fcdc02954722f37f884cb70a75cda03 Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Fri, 22 Jan 2021 13:05:58 -0700 Subject: [PATCH 09/74] Add support for additional breakdowns of daily cost data Hide expansion option if not enough breakdowns --- .changeset/cost-insights-yellow-trees-love.md | 6 + plugins/cost-insights/src/client.ts | 14 ++- ...art.tsx => CostOverviewBreakdownChart.tsx} | 115 +++++++++--------- .../CostOverviewCard.test.tsx | 89 ++++++++++++++ .../CostOverviewCard/CostOverviewCard.tsx | 45 ++++--- plugins/cost-insights/src/types/Cost.ts | 2 +- plugins/cost-insights/src/utils/mockData.ts | 15 +++ 7 files changed, 209 insertions(+), 77 deletions(-) create mode 100644 .changeset/cost-insights-yellow-trees-love.md rename plugins/cost-insights/src/components/CostOverviewCard/{CostOverviewByProductChart.tsx => CostOverviewBreakdownChart.tsx} (67%) create mode 100644 plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.test.tsx diff --git a/.changeset/cost-insights-yellow-trees-love.md b/.changeset/cost-insights-yellow-trees-love.md new file mode 100644 index 0000000000..19eea0dfe2 --- /dev/null +++ b/.changeset/cost-insights-yellow-trees-love.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-cost-insights': minor +--- + +Add support for additional breakdowns of daily cost data. +This changes the type of Cost.groupedCosts returned by CostInsightsApi.getGroupDailyCost. diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index 0ba57f76fa..c43deba6ae 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -33,11 +33,12 @@ import { UnlabeledDataflowAlert, } from '../src/utils/alerts'; import { - trendlineOf, + aggregationFor, changeOf, entityOf, getGroupedProducts, - aggregationFor, + getGroupedProjects, + trendlineOf, } from './utils/mockData'; export class ExampleCostInsightsClient implements CostInsightsApi { @@ -101,7 +102,10 @@ export class ExampleCostInsightsClient implements CostInsightsApi { trendline: trendlineOf(aggregation), // Optional field on Cost which needs to be supplied in order to see // the product breakdown view in the top panel. - groupedCosts: getGroupedProducts(intervals), + groupedCosts: { + product: getGroupedProducts(intervals), + project: getGroupedProjects(intervals), + }, }, ); @@ -119,7 +123,9 @@ export class ExampleCostInsightsClient implements CostInsightsApi { trendline: trendlineOf(aggregation), // Optional field on Cost which needs to be supplied in order to see // the product breakdown view in the top panel. - groupedCosts: getGroupedProducts(intervals), + groupedCosts: { + product: getGroupedProducts(intervals), + }, }, ); diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewByProductChart.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewBreakdownChart.tsx similarity index 67% rename from plugins/cost-insights/src/components/CostOverviewCard/CostOverviewByProductChart.tsx rename to plugins/cost-insights/src/components/CostOverviewCard/CostOverviewBreakdownChart.tsx index b562434c5e..feec23933f 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewByProductChart.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewBreakdownChart.tsx @@ -56,26 +56,26 @@ import { BarChartLegendOptions } from '../BarChart/BarChartLegend'; dayjs.extend(utc); -export type CostOverviewByProductChartProps = { - costsByProduct: Cost[]; +export type CostOverviewBreakdownChartProps = { + costBreakdown: Cost[]; }; const LOW_COST_THRESHOLD = 0.1; -export const CostOverviewByProductChart = ({ - costsByProduct, -}: CostOverviewByProductChartProps) => { +export const CostOverviewBreakdownChart = ({ + costBreakdown, +}: CostOverviewBreakdownChartProps) => { const theme = useTheme(); const classes = useStyles(theme); const lastCompleteBillingDate = useLastCompleteBillingDate(); const { duration } = useFilters(mapFiltersToProps); const [isExpanded, setExpanded] = useState(false); - if (!costsByProduct) { + if (!costBreakdown) { return null; } - const flattenedAggregation = costsByProduct + const flattenedAggregation = costBreakdown .map(cost => cost.aggregation) .flat(); @@ -87,44 +87,46 @@ export const CostOverviewByProductChart = ({ lastCompleteBillingDate, ); const currentPeriodTotal = totalCost - previousPeriodTotal; - const otherProducts: string[] = []; + const canExpand = costBreakdown.length >= 8; + const otherCategoryIds: string[] = []; - const productsByDate = costsByProduct.reduce((prodByDate, product) => { - const productTotal = aggregationSum(product.aggregation); - // Group products with less than 10% of the total cost into "Other" category - // when we have >= 8 products. - const isOtherProduct = - costsByProduct.length >= 8 && - productTotal < totalCost * LOW_COST_THRESHOLD; + const breakdownsByDate = costBreakdown.reduce( + (breakdownByDate, breakdown) => { + const breakdownTotal = aggregationSum(breakdown.aggregation); + // Group breakdown items with less than 10% of the total cost into "Other" category if needed + const isOtherCategory = + canExpand && breakdownTotal < totalCost * LOW_COST_THRESHOLD; - const updatedProdByDate = { ...prodByDate }; - if (isOtherProduct) { - otherProducts.push(product.id); - } - product.aggregation.forEach(curAggregation => { - const productCostsForDate = updatedProdByDate[curAggregation.date] || {}; + const updatedBreakdownByDate = { ...breakdownByDate }; + if (isOtherCategory) { + otherCategoryIds.push(breakdown.id); + } + breakdown.aggregation.forEach(curAggregation => { + const costsForDate = updatedBreakdownByDate[curAggregation.date] || {}; - updatedProdByDate[curAggregation.date] = { - ...productCostsForDate, - [product.id]: - (productCostsForDate[product.id] || 0) + curAggregation.amount, - }; - }); + updatedBreakdownByDate[curAggregation.date] = { + ...costsForDate, + [breakdown.id]: + (costsForDate[breakdown.id] || 0) + curAggregation.amount, + }; + }); - return updatedProdByDate; - }, {} as Record>); + return updatedBreakdownByDate; + }, + {} as Record>, + ); - const chartData: Record[] = Object.keys(productsByDate).map( + const chartData: Record[] = Object.keys(breakdownsByDate).map( date => { - const costsForDate = Object.keys(productsByDate[date]).reduce( - (dateCosts, product) => { - // Group costs for products that belong to 'Other' in the chart. - const cost = productsByDate[date][product]; - const productCost = - !isExpanded && otherProducts.includes(product) + const costsForDate = Object.keys(breakdownsByDate[date]).reduce( + (dateCosts, breakdown) => { + // Group costs for items that belong to 'Other' in the chart. + const cost = breakdownsByDate[date][breakdown]; + const breakdownCost = + !isExpanded && otherCategoryIds.includes(breakdown) ? { Other: (dateCosts.Other || 0) + cost } - : { [product]: cost }; - return { ...dateCosts, ...productCost }; + : { [breakdown]: cost }; + return { ...dateCosts, ...breakdownCost }; }, {} as Record, ); @@ -135,40 +137,41 @@ export const CostOverviewByProductChart = ({ }, ); - const sortedProducts = costsByProduct.sort( + const sortedBreakdowns = costBreakdown.sort( (a, b) => aggregationSum(a.aggregation) - aggregationSum(b.aggregation), ); const renderAreas = () => { - const separatedProducts = sortedProducts - // Check that product is a separate group and hasn't been added to 'Other' + const separatedBreakdowns = sortedBreakdowns + // Check that the breakdown is a separate group and hasn't been added to 'Other' .filter( - product => - product.id !== 'Other' && !otherProducts.includes(product.id), + breakdown => + breakdown.id !== 'Other' && !otherCategoryIds.includes(breakdown.id), ) - .map(product => product.id); + .map(breakdown => breakdown.id); // Keep 'Other' category at the bottom of the stack - const productsToDisplay = isExpanded - ? sortedProducts.map(product => product.id) - : ['Other', ...separatedProducts]; + const breakdownsToDisplay = isExpanded + ? sortedBreakdowns.map(breakdown => breakdown.id) + : ['Other', ...separatedBreakdowns]; - return productsToDisplay.map((product, i) => { - // Logic to handle case where there are more products than data viz colors. - const productColor = + return breakdownsToDisplay.map((breakdown, i) => { + // Logic to handle case where there are more items than data viz colors. + const color = theme.palette.dataViz[ - (productsToDisplay.length - 1 - i) % + (breakdownsToDisplay.length - 1 - i) % (theme.palette.dataViz.length - 1) ]; return ( setExpanded(true)} style={{ - cursor: product === 'Other' && !isExpanded ? 'pointer' : null, + cursor: breakdown === 'Other' && !isExpanded ? 'pointer' : null, }} /> ); @@ -206,7 +209,7 @@ export const CostOverviewByProductChart = ({ {items.reverse().map((item, index) => ( ))} - {!isExpanded ? expandText : null} + {canExpand && !isExpanded ? expandText : null} ); }; diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.test.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.test.tsx new file mode 100644 index 0000000000..6fca8f67c9 --- /dev/null +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { fireEvent } from '@testing-library/react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { CostOverviewCard } from './CostOverviewCard'; +import { Cost } from '../../types'; +import { + changeOf, + getGroupedProducts, + getGroupedProjects, + MockAggregatedDailyCosts, + trendlineOf, +} from '../../utils/mockData'; +import { + MockBillingDateProvider, + MockConfigProvider, + MockFilterProvider, + MockScrollProvider, +} from '../../utils/tests'; +import { CostInsightsThemeProvider } from '../CostInsightsPage/CostInsightsThemeProvider'; + +const mockGroupDailyCost: Cost = { + id: 'test-group', + aggregation: MockAggregatedDailyCosts, + change: changeOf(MockAggregatedDailyCosts), + trendline: trendlineOf(MockAggregatedDailyCosts), +}; + +function renderInContext(children: JSX.Element) { + return renderInTestApp( + + + + + {children} + + + + , + ); +} + +describe('', () => { + it('Renders without exploding', async () => { + const { getByText } = await renderInContext( + , + ); + expect(getByText('Cloud Cost')).toBeInTheDocument(); + }); + + it('Shows breakdown tabs if provided', async () => { + const mockDailyCostWithBreakdowns = { + ...mockGroupDailyCost, + groupedCosts: { + product: getGroupedProducts('R2/P90D/2021-01-01'), + project: getGroupedProjects('R2/P90D/2021-01-01'), + }, + }; + const { getByText } = await renderInContext( + , + ); + expect(getByText('Cloud Cost')).toBeInTheDocument(); + expect(getByText('Breakdown by product')).toBeInTheDocument(); + expect(getByText('Breakdown by project')).toBeInTheDocument(); + + fireEvent.click(getByText('Breakdown by product')); + expect(getByText('Cloud Cost By Product')).toBeInTheDocument(); + + fireEvent.click(getByText('Breakdown by project')); + expect(getByText('Cloud Cost By Project')).toBeInTheDocument(); + }); +}); diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx index 39adc4bbf1..614e045b20 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx @@ -14,22 +14,22 @@ * limitations under the License. */ -import React, { useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { Box, Card, CardContent, Divider, - useTheme, Tab, Tabs, + useTheme, } from '@material-ui/core'; import { CostOverviewChart } from './CostOverviewChart'; -import { CostOverviewByProductChart } from './CostOverviewByProductChart'; +import { CostOverviewBreakdownChart } from './CostOverviewBreakdownChart'; import { CostOverviewHeader } from './CostOverviewHeader'; import { MetricSelect } from '../MetricSelect'; import { PeriodSelect } from '../PeriodSelect'; -import { useScroll, useFilters, useConfig } from '../../hooks'; +import { useConfig, useFilters, useScroll } from '../../hooks'; import { mapFiltersToProps } from './selector'; import { DefaultNavigation } from '../../utils/navigation'; import { findAlways } from '../../utils/assert'; @@ -49,6 +49,15 @@ export const CostOverviewCard = ({ const config = useConfig(); const [tabIndex, setTabIndex] = useState(0); + // Reset tabIndex if breakdowns available change + useEffect(() => { + // Intentionally off-by-one to account for the overview tab + const lastIndex = Object.keys(dailyCostData.groupedCosts ?? {}).length; + if (tabIndex > lastIndex) { + setTabIndex(0); + } + }, [dailyCostData, tabIndex, setTabIndex]); + const { ScrollAnchor } = useScroll(DefaultNavigation.CostOverviewCard); const { setDuration, setProject, setMetric, ...filters } = useFilters( mapFiltersToProps, @@ -59,14 +68,18 @@ export const CostOverviewCard = ({ : null; const styles = useOverviewTabsStyles(theme); + const breakdownTabs = Object.keys(dailyCostData.groupedCosts ?? {}).map( + key => ({ + id: key, + label: `Breakdown by ${key}`, + title: `Cloud Cost By ${key.charAt(0).toUpperCase() + key.slice(1)}`, + }), + ); const tabs = [ { id: 'overview', label: 'Total cost', title: 'Cloud Cost' }, - { - id: 'breakdown', - label: 'Breakdown by product', - title: 'Cloud Cost By Product', - }, - ]; + ].concat(breakdownTabs); + // tabIndex can temporarily be invalid while the useEffect above processes + const safeTabIndex = tabIndex > tabs.length - 1 ? 0 : tabIndex; const OverviewTabs = () => { return ( @@ -74,7 +87,7 @@ export const CostOverviewCard = ({ setTabIndex(index)} - value={tabIndex} + value={safeTabIndex} > {tabs.map((tab, index) => ( {dailyCostData.groupedCosts && } - + - {tabIndex === 0 ? ( + {safeTabIndex === 0 ? ( ) : ( - )} diff --git a/plugins/cost-insights/src/types/Cost.ts b/plugins/cost-insights/src/types/Cost.ts index dd28bba6b0..c3b63946af 100644 --- a/plugins/cost-insights/src/types/Cost.ts +++ b/plugins/cost-insights/src/types/Cost.ts @@ -23,5 +23,5 @@ export interface Cost { aggregation: DateAggregation[]; change?: ChangeStatistic; trendline?: Trendline; - groupedCosts?: Cost[]; + groupedCosts?: Record; } diff --git a/plugins/cost-insights/src/utils/mockData.ts b/plugins/cost-insights/src/utils/mockData.ts index f84473b84f..14ff9aecd1 100644 --- a/plugins/cost-insights/src/utils/mockData.ts +++ b/plugins/cost-insights/src/utils/mockData.ts @@ -1064,3 +1064,18 @@ export const getGroupedProducts = (intervals: string) => [ aggregation: aggregationFor(intervals, 250), }, ]; + +export const getGroupedProjects = (intervals: string) => [ + { + id: 'project-a', + aggregation: aggregationFor(intervals, 1_700), + }, + { + id: 'project-b', + aggregation: aggregationFor(intervals, 350), + }, + { + id: 'project-c', + aggregation: aggregationFor(intervals, 1_300), + }, +]; From a91aa6bf2a32a96eacae41e550486882b4a832b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 25 Jan 2021 15:05:18 +0100 Subject: [PATCH 10/74] catalog: support supplying a custom catalog descriptor file parser --- .changeset/green-boats-attend.md | 5 ++++ .../src/ingestion/LocationReaders.ts | 4 +++- .../processors/UrlReaderProcessor.test.ts | 5 ++-- .../processors/UrlReaderProcessor.ts | 10 +++++--- .../src/ingestion/processors/types.ts | 13 +++++++++++ .../src/ingestion/processors/util/parse.ts | 11 ++++++++- .../src/service/CatalogBuilder.test.ts | 23 +++++++++++++++++++ .../src/service/CatalogBuilder.ts | 20 ++++++++++++++++ 8 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 .changeset/green-boats-attend.md diff --git a/.changeset/green-boats-attend.md b/.changeset/green-boats-attend.md new file mode 100644 index 0000000000..ae2324291e --- /dev/null +++ b/.changeset/green-boats-attend.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Support supplying a custom catalog descriptor file parser diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index cb703b8e15..046b6bdf76 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -32,6 +32,7 @@ import { CatalogProcessorEntityResult, CatalogProcessorErrorResult, CatalogProcessorLocationResult, + CatalogProcessorParser, CatalogProcessorResult, } from './processors/types'; import { LocationReader, ReadLocationResult } from './types'; @@ -41,6 +42,7 @@ const MAX_DEPTH = 10; type Options = { reader: UrlReader; + parser: CatalogProcessorParser; logger: Logger; config: Config; processors: CatalogProcessor[]; @@ -137,7 +139,6 @@ export class LocationReaders implements LocationReader { if (emitResult.type === 'relation') { throw new Error('readLocation may not emit entity relations'); } - emit(emitResult); }; @@ -149,6 +150,7 @@ export class LocationReaders implements LocationReader { item.location, item.optional, validatedEmit, + this.options.parser, ) ) { return; diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts index aac4235e3a..49b6fe71e0 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts @@ -25,6 +25,7 @@ import { CatalogProcessorErrorResult, CatalogProcessorResult, } from './types'; +import { defaultEntityDataParser } from './util/parse'; describe('UrlReaderProcessor', () => { const mockApiOrigin = 'http://localhost'; @@ -52,7 +53,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorEntityResult; expect(generated.type).toBe('entity'); @@ -81,7 +82,7 @@ describe('UrlReaderProcessor', () => { ); const generated = (await new Promise(emit => - processor.readLocation(spec, false, emit), + processor.readLocation(spec, false, emit, defaultEntityDataParser), )) as CatalogProcessorErrorResult; expect(generated.type).toBe('error'); diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index 04f5ee738a..9daae29d3b 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -18,8 +18,11 @@ import { UrlReader } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { Logger } from 'winston'; import * as result from './results'; -import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { parseEntityYaml } from './util/parse'; +import { + CatalogProcessor, + CatalogProcessorEmit, + CatalogProcessorParser, +} from './types'; // TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this // can be removed in a bit @@ -43,6 +46,7 @@ export class UrlReaderProcessor implements CatalogProcessor { location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise { if (deprecatedTypes.includes(location.type)) { // TODO(Rugvip): Remove this warning a month or two into 2021, and remove support for the deprecated types. @@ -57,7 +61,7 @@ export class UrlReaderProcessor implements CatalogProcessor { try { const data = await this.options.reader.read(location.target); - for (const parseResult of parseEntityYaml(data, location)) { + for await (const parseResult of parser({ data, location })) { emit(parseResult); } } catch (error) { diff --git a/plugins/catalog-backend/src/ingestion/processors/types.ts b/plugins/catalog-backend/src/ingestion/processors/types.ts index 1bf30567bc..f7e11d5616 100644 --- a/plugins/catalog-backend/src/ingestion/processors/types.ts +++ b/plugins/catalog-backend/src/ingestion/processors/types.ts @@ -27,12 +27,15 @@ export type CatalogProcessor = { * @param location The location to read * @param optional Whether a missing target should trigger an error * @param emit A sink for items resulting from the read + * @param parser A parser, that is able to take the raw catalog descriptor + * data and turn it into the actual result pieces. * @returns True if handled by this processor, false otherwise */ readLocation?( location: LocationSpec, optional: boolean, emit: CatalogProcessorEmit, + parser: CatalogProcessorParser, ): Promise; /** @@ -100,6 +103,16 @@ export type CatalogProcessor = { ): Promise; }; +/** + * A parser, that is able to take the raw catalog descriptor data and turn it + * into the actual result pieces. The default implementation performs a YAML + * document parsing. + */ +export type CatalogProcessorParser = (options: { + data: Buffer; + location: LocationSpec; +}) => AsyncIterable; + export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void; export type CatalogProcessorLocationResult = { diff --git a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts index c3dcd42d62..aa24968d6d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/parse.ts +++ b/plugins/catalog-backend/src/ingestion/processors/util/parse.ts @@ -18,7 +18,7 @@ import { Entity, LocationSpec } from '@backstage/catalog-model'; import lodash from 'lodash'; import yaml from 'yaml'; import * as result from '../results'; -import { CatalogProcessorResult } from '../types'; +import { CatalogProcessorParser, CatalogProcessorResult } from '../types'; export function* parseEntityYaml( data: Buffer, @@ -50,3 +50,12 @@ export function* parseEntityYaml( } } } + +export const defaultEntityDataParser: CatalogProcessorParser = async function* defaultEntityDataParser({ + data, + location, +}) { + for (const e of parseEntityYaml(data, location)) { + yield e; + } +}; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 37337f76f1..695902fec9 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -20,6 +20,7 @@ import { ConfigReader } from '@backstage/config'; import Knex from 'knex'; import yaml from 'yaml'; import { DatabaseManager } from '../database'; +import { CatalogProcessorParser } from '../ingestion'; import * as result from '../ingestion/processors/results'; import { CatalogBuilder, CatalogEnvironment } from './CatalogBuilder'; @@ -209,4 +210,26 @@ describe('CatalogBuilder', () => { }), ]); }); + + it('setEntityDataParser works', async () => { + const mockParser: CatalogProcessorParser = jest + .fn() + .mockImplementation(() => {}); + + const builder = new CatalogBuilder(env) + .setEntityDataParser(mockParser) + .replaceProcessors([ + { + async readLocation(_location, _optional, _emit, parser) { + expect(parser).toBe(mockParser); + return true; + }, + }, + ]); + + const { higherOrderOperation } = await builder.build(); + await higherOrderOperation.addLocation({ type: 'x', target: 'y' }); + + expect.assertions(1); + }); }); diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index f6170135fd..938a406e56 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -39,6 +39,7 @@ import { AnnotateLocationEntityProcessor, BuiltinKindsEntityProcessor, CatalogProcessor, + CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, GithubOrgReaderProcessor, @@ -60,6 +61,7 @@ import { textPlaceholderResolver, yamlPlaceholderResolver, } from '../ingestion/processors/PlaceholderProcessor'; +import { defaultEntityDataParser } from '../ingestion/processors/util/parse'; import { LocationAnalyzer } from '../ingestion/types'; export type CatalogEnvironment = { @@ -96,6 +98,7 @@ export class CatalogBuilder { private fieldFormatValidators: Partial; private processors: CatalogProcessor[]; private processorsReplace: boolean; + private parser: CatalogProcessorParser | undefined; constructor(env: CatalogEnvironment) { this.env = env; @@ -105,6 +108,7 @@ export class CatalogBuilder { this.fieldFormatValidators = {}; this.processors = []; this.processorsReplace = false; + this.parser = undefined; } /** @@ -197,6 +201,20 @@ export class CatalogBuilder { return this; } + /** + * Sets up the catalog to use a custom parser for entity data. + * + * This is the function that gets called immediately after some raw entity + * specification data has been read from a remote source, and needs to be + * parsed and emitted as structured data. + * + * @param parser The custom parser + */ + setEntityDataParser(parser: CatalogProcessorParser): CatalogBuilder { + this.parser = parser; + return this; + } + /** * Wires up and returns all of the component parts of the catalog */ @@ -211,9 +229,11 @@ export class CatalogBuilder { const policy = this.buildEntityPolicy(); const processors = this.buildProcessors(); const rulesEnforcer = CatalogRulesEnforcer.fromConfig(config); + const parser = this.parser || defaultEntityDataParser; const locationReader = new LocationReaders({ ...this.env, + parser, processors, rulesEnforcer, policy, From 38f710b366ffec2f3ef78d74d6fcc78de8a0bef0 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Mon, 25 Jan 2021 16:10:50 +0100 Subject: [PATCH 11/74] Fix commits url for bitbucket server --- .../src/reading/BitbucketUrlReader.ts | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index e9727e04cf..86dd7e32b0 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -161,13 +161,18 @@ export class BitbucketUrlReader implements UrlReader { } private async getLastCommitShortHash(url: string): Promise { - const { name: repoName, owner: project, ref } = parseGitUrl(url); + const { resource, name: repoName, owner: project, ref } = parseGitUrl(url); let branch = ref; if (!branch) { branch = await getBitbucketDefaultBranch(url, this.config); } - const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`; + + const isHosted = resource === 'bitbucket.org'; + // Bitbucket Server https://docs.atlassian.com/bitbucket-server/rest/7.9.0/bitbucket-rest.html#idp222 + const commitsApiUrl = isHosted + ? `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}` + : `${this.config.apiBaseUrl}/projects/${project}/repos/${repoName}/commits`; const commitsResponse = await fetch( commitsApiUrl, @@ -182,14 +187,26 @@ export class BitbucketUrlReader implements UrlReader { } const commits = await commitsResponse.json(); - if ( - commits && - commits.values && - commits.values.length > 0 && - commits.values[0].hash - ) { - return commits.values[0].hash.substring(0, 12); + if (isHosted) { + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].hash + ) { + return commits.values[0].hash.substring(0, 12); + } + } else { + if ( + commits && + commits.values && + commits.values.length > 0 && + commits.values[0].id + ) { + return commits.values[0].id.substring(0, 12); + } } + throw new Error(`Failed to read response from ${commitsApiUrl}`); } } From 31e7b2a1fde62ed8e175345862cef1fee936a834 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Mon, 25 Jan 2021 16:26:02 +0100 Subject: [PATCH 12/74] Fix tests using wrong commits url for bitbucket server --- .../backend-common/src/reading/BitbucketUrlReader.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 9661368b5e..6265718571 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -126,12 +126,12 @@ describe('BitbucketUrlReader', () => { ), ), rest.get( - 'https://api.bitbucket.mycompany.net/rest/api/1.0/repositories/backstage/mock/commits/some-branch', + 'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/commits', (_, res, ctx) => res( ctx.status(200), ctx.json({ - values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], + values: [{ id: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }], }), ), ), From 9ba282f7c7bd402a64dcc79c5287c824df0505e6 Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Mon, 25 Jan 2021 20:15:50 -0700 Subject: [PATCH 13/74] Update comments, use MUI's capitalize --- plugins/cost-insights/src/client.ts | 8 ++++---- .../src/components/CostOverviewCard/CostOverviewCard.tsx | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/plugins/cost-insights/src/client.ts b/plugins/cost-insights/src/client.ts index c43deba6ae..fa9a1d23f5 100644 --- a/plugins/cost-insights/src/client.ts +++ b/plugins/cost-insights/src/client.ts @@ -100,8 +100,8 @@ export class ExampleCostInsightsClient implements CostInsightsApi { aggregation: aggregation, change: changeOf(aggregation), trendline: trendlineOf(aggregation), - // Optional field on Cost which needs to be supplied in order to see - // the product breakdown view in the top panel. + // Optional field providing cost groupings / breakdowns keyed by the type. In this example, + // daily cost grouped by cloud product OR by project / billing account. groupedCosts: { product: getGroupedProducts(intervals), project: getGroupedProjects(intervals), @@ -121,8 +121,8 @@ export class ExampleCostInsightsClient implements CostInsightsApi { aggregation: aggregation, change: changeOf(aggregation), trendline: trendlineOf(aggregation), - // Optional field on Cost which needs to be supplied in order to see - // the product breakdown view in the top panel. + // Optional field providing cost groupings / breakdowns keyed by the type. In this example, + // daily project cost grouped by cloud product. groupedCosts: { product: getGroupedProducts(intervals), }, diff --git a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx index 614e045b20..56414c9a0e 100644 --- a/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx +++ b/plugins/cost-insights/src/components/CostOverviewCard/CostOverviewCard.tsx @@ -17,6 +17,7 @@ import React, { useEffect, useState } from 'react'; import { Box, + capitalize, Card, CardContent, Divider, @@ -72,7 +73,7 @@ export const CostOverviewCard = ({ key => ({ id: key, label: `Breakdown by ${key}`, - title: `Cloud Cost By ${key.charAt(0).toUpperCase() + key.slice(1)}`, + title: `Cloud Cost By ${capitalize(key)}`, }), ); const tabs = [ From 6327bb8770a78c8696928d885c8f302cfc70cbc9 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 13:33:56 +0100 Subject: [PATCH 14/74] chore: Added example multi-stage with the new backstage bundle command --- .../docker/multi-stage-with-bundle/Dockerfile | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 contrib/docker/multi-stage-with-bundle/Dockerfile diff --git a/contrib/docker/multi-stage-with-bundle/Dockerfile b/contrib/docker/multi-stage-with-bundle/Dockerfile new file mode 100644 index 0000000000..5e6ce770de --- /dev/null +++ b/contrib/docker/multi-stage-with-bundle/Dockerfile @@ -0,0 +1,42 @@ +# Stage 1 - Create yarn install skeleton layer +FROM node:14-buster AS packages + +WORKDIR /app +COPY package.json yarn.lock ./ + +# Uncomment this line if building a non create-app version +# COPY packages packages +COPY plugins plugins + +RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -print | xargs rm -rf + +# Stage 2 - Install dependencies and build packages +FROM node:14-buster AS build + +WORKDIR /app +COPY --from=packages /app . + +RUN yarn install --network-timeout 600000 && rm -rf "$(yarn cache dir)" + +COPY . . + +RUN yarn tsc +RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies + +# Stage 3 - Build the actual backend image and install production dependencies +FROM node:14-buster + +WORKDIR /app + +# Copy from build stage +COPY --from=build /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton.tar.gz ./ +RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz + +RUN yarn install --production --network-timeout 600000 && rm -rf "$(yarn cache dir)" + +COPY --from=build /app/packages/backend/dist/bundle.tar.gz . +RUN tar xzf bundle.tar.gz && rm bundle.tar.gz + +COPY app-config.yaml app-config.production.yaml ./ + +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] \ No newline at end of file From 294b7b087d222112134c914ca6222c37fc231ffd Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 13:41:15 +0100 Subject: [PATCH 15/74] chore: fix newline on new file --- contrib/docker/multi-stage-with-bundle/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/multi-stage-with-bundle/Dockerfile b/contrib/docker/multi-stage-with-bundle/Dockerfile index 5e6ce770de..8289a49046 100644 --- a/contrib/docker/multi-stage-with-bundle/Dockerfile +++ b/contrib/docker/multi-stage-with-bundle/Dockerfile @@ -39,4 +39,4 @@ RUN tar xzf bundle.tar.gz && rm bundle.tar.gz COPY app-config.yaml app-config.production.yaml ./ -CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] \ No newline at end of file +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] From 00448abb0343cbab1ff8aa3ce6393ec0c3688158 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 15:47:23 +0100 Subject: [PATCH 16/74] chore: updating docs reference to integrations condfig --- .../software-templates/installation.md | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 724b86aed1..6d516eef3a 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -173,7 +173,12 @@ and access to a running Docker daemon. You can create a GitHub access token docs on creating private GitHub access tokens is available [here](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token). Note that the need for private GitHub access tokens will be replaced with GitHub -Apps integration further down the line. +Apps integration further down the line by using the existing `integrations` +config. + +> Note: Some of this config may already be setup part of your `app-config.yaml`. +> We're moving away from the duplciated config for authentication in the +> `scaffolder` section and using `integrations` instead. #### GitHub @@ -187,11 +192,14 @@ by specifying `visibility` option. Valid options are `public`, `private` and public within the enterprise. ```yaml -scaffolder: +integrations: github: - token: - $env: GITHUB_TOKEN - visibility: public # or 'internal' or 'private' + - host: github.com + token: + $env: GITHUB_TOKEN + +scaffolder: + visibility: public # or 'internal' or 'private' ``` #### GitLab @@ -201,10 +209,9 @@ allows to configure the private access token and the base URL of a GitLab instance: ```yaml -scaffolder: +integrations: gitlab: - api: - baseUrl: https://gitlab.com + - host: gitlab.com token: $env: GITLAB_TOKEN ``` @@ -218,10 +225,9 @@ will hopefully support on-prem installations as well but that has not been verified. ```yaml -scaffolder: +integrations: azure: - baseUrl: https://dev.azure.com/{your-organization} - api: + - host: dev.azure.com token: $env: AZURE_TOKEN ``` From 46533ed01b75c8751f6c7c05ffb1794098f8cd2c Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 15:53:32 +0100 Subject: [PATCH 17/74] chore: fix the config key for scaffolderr --- docs/features/software-templates/installation.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index 6d516eef3a..c2f95f26ea 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -199,7 +199,8 @@ integrations: $env: GITHUB_TOKEN scaffolder: - visibility: public # or 'internal' or 'private' + github: + visibility: public # or 'internal' or 'private' ``` #### GitLab From ad52b78b8dd50331cf535a461a1e3ae51cacd97f Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 15:55:34 +0100 Subject: [PATCH 18/74] chore: reworking the files so that it's moved to docs instea --- .../docker/multi-stage-with-bundle/Dockerfile | 42 ------------ docs/getting-started/deployment-other.md | 64 ++++++++++++++++--- 2 files changed, 56 insertions(+), 50 deletions(-) delete mode 100644 contrib/docker/multi-stage-with-bundle/Dockerfile diff --git a/contrib/docker/multi-stage-with-bundle/Dockerfile b/contrib/docker/multi-stage-with-bundle/Dockerfile deleted file mode 100644 index 8289a49046..0000000000 --- a/contrib/docker/multi-stage-with-bundle/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Stage 1 - Create yarn install skeleton layer -FROM node:14-buster AS packages - -WORKDIR /app -COPY package.json yarn.lock ./ - -# Uncomment this line if building a non create-app version -# COPY packages packages -COPY plugins plugins - -RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -print | xargs rm -rf - -# Stage 2 - Install dependencies and build packages -FROM node:14-buster AS build - -WORKDIR /app -COPY --from=packages /app . - -RUN yarn install --network-timeout 600000 && rm -rf "$(yarn cache dir)" - -COPY . . - -RUN yarn tsc -RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies - -# Stage 3 - Build the actual backend image and install production dependencies -FROM node:14-buster - -WORKDIR /app - -# Copy from build stage -COPY --from=build /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton.tar.gz ./ -RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz - -RUN yarn install --production --network-timeout 600000 && rm -rf "$(yarn cache dir)" - -COPY --from=build /app/packages/backend/dist/bundle.tar.gz . -RUN tar xzf bundle.tar.gz && rm bundle.tar.gz - -COPY app-config.yaml app-config.production.yaml ./ - -CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 55582ee77a..900173a6c5 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -4,19 +4,67 @@ title: Other description: Documentation on different ways of Deployment --- -## Deploying Locally +## Docker -### Try on Docker +Here we have an example Dockerfile that you can use to build everything together +in one container. This Dockerfile uses multi-stage builds, and a +`backend:bundle` command from the CLI. -Run the following commands if you have Docker environment +It also provides caching on the `yarn install`'s so that you don't have to do it +unless absolutely necessary. -```bash -$ yarn install -$ yarn docker-build -$ docker run --rm -it -p 7000:7000 -e APP_ENV=production -e NODE_ENV=development example-backend:latest +```Dockerfile +# Stage 1 - Create yarn install skeleton layer +FROM node:14-buster AS packages + +WORKDIR /app +COPY package.json yarn.lock ./ + +COPY packages packages + +# Uncomment this line if you have a local plugins folder +# COPY plugins plugins + +RUN find packages \! -name "package.json" -mindepth 2 -maxdepth 2 -print | xargs rm -rf + +# Stage 2 - Install dependencies and build packages +FROM node:14-buster AS build + +WORKDIR /app +COPY --from=packages /app . + +RUN yarn install --network-timeout 600000 && rm -rf "$(yarn cache dir)" + +COPY . . + +RUN yarn tsc +RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies + +# Stage 3 - Build the actual backend image and install production dependencies +FROM node:14-buster + +WORKDIR /app + +# Copy from build stage +COPY --from=build /app/yarn.lock /app/package.json /app/packages/backend/dist/skeleton.tar.gz ./ +RUN tar xzf skeleton.tar.gz && rm skeleton.tar.gz + +RUN yarn install --production --network-timeout 600000 && rm -rf "$(yarn cache dir)" + +COPY --from=build /app/packages/backend/dist/bundle.tar.gz . +RUN tar xzf bundle.tar.gz && rm bundle.tar.gz + +COPY app-config.yaml app-config.production.yaml ./ + +CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] ``` -Then open http://localhost:7000 on your browser. +You can add the Dockerfile to the root of your project, and run the following: + +```sh +$ docker build -t eaxmple-deployment . +$ docker run -p 7000:7000 example-deployment +``` ## Heroku From 6cc06d4e7f7630c6747b2a8bc0fa884cb7637366 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 15:56:33 +0100 Subject: [PATCH 19/74] chore: fixing spelling of docs --- docs/getting-started/deployment-other.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 900173a6c5..6cc90c88fb 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -62,7 +62,7 @@ CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app You can add the Dockerfile to the root of your project, and run the following: ```sh -$ docker build -t eaxmple-deployment . +$ docker build -t example-deployment . $ docker run -p 7000:7000 example-deployment ``` From 2f547a668fea41611515737741136ebd49ecfc26 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 16:03:26 +0100 Subject: [PATCH 20/74] chore: fix spelling mistake --- docs/features/software-templates/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index c2f95f26ea..c0bc10fbe0 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -177,7 +177,7 @@ Apps integration further down the line by using the existing `integrations` config. > Note: Some of this config may already be setup part of your `app-config.yaml`. -> We're moving away from the duplciated config for authentication in the +> We're moving away from the duplicated config for authentication in the > `scaffolder` section and using `integrations` instead. #### GitHub From e77a8dc14e905e728bcb685cbeee56fad73d01d6 Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Tue, 26 Jan 2021 17:02:03 +0100 Subject: [PATCH 21/74] Update docs/features/software-templates/installation.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw --- docs/features/software-templates/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index c0bc10fbe0..a00de1aa5e 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -176,7 +176,7 @@ Note that the need for private GitHub access tokens will be replaced with GitHub Apps integration further down the line by using the existing `integrations` config. -> Note: Some of this config may already be setup part of your `app-config.yaml`. +> Note: Some of this configuration may already be set up as part of your `app-config.yaml`. > We're moving away from the duplicated config for authentication in the > `scaffolder` section and using `integrations` instead. From cb99b8fc1dcbb8b08e531e4842258ac9eda39775 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 17:07:27 +0100 Subject: [PATCH 22/74] chore: prettier makes pretty --- docs/features/software-templates/installation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/features/software-templates/installation.md b/docs/features/software-templates/installation.md index a00de1aa5e..8d2fa7727d 100644 --- a/docs/features/software-templates/installation.md +++ b/docs/features/software-templates/installation.md @@ -176,9 +176,9 @@ Note that the need for private GitHub access tokens will be replaced with GitHub Apps integration further down the line by using the existing `integrations` config. -> Note: Some of this configuration may already be set up as part of your `app-config.yaml`. -> We're moving away from the duplicated config for authentication in the -> `scaffolder` section and using `integrations` instead. +> Note: Some of this configuration may already be set up as part of your +> `app-config.yaml`. We're moving away from the duplicated config for +> authentication in the `scaffolder` section and using `integrations` instead. #### GitHub From 2dee00292cbdf35d52b43401cea738eaded29e36 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 17:34:52 +0100 Subject: [PATCH 23/74] chore: review comments --- docs/getting-started/deployment-other.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 6cc90c88fb..569ef6e898 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -13,6 +13,9 @@ in one container. This Dockerfile uses multi-stage builds, and a It also provides caching on the `yarn install`'s so that you don't have to do it unless absolutely necessary. +> Note: This Dockerfile assumes that you're running SQLite, or your +> configuration is setup to connect to an external PostgreSQL Database. + ```Dockerfile # Stage 1 - Create yarn install skeleton layer FROM node:14-buster AS packages @@ -66,6 +69,8 @@ $ docker build -t example-deployment . $ docker run -p 7000:7000 example-deployment ``` +Once complete, open your browser at `http://localhost:7000` if running locally. + ## Heroku Deploying to Heroku is relatively easy following these steps. From 8080d1ef4fb0b263f08568d605406ce6c8d6728a Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 17:39:28 +0100 Subject: [PATCH 24/74] chore: change the wording slightly --- docs/getting-started/deployment-other.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 569ef6e898..6436bdb195 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -62,14 +62,21 @@ COPY app-config.yaml app-config.production.yaml ./ CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] ``` -You can add the Dockerfile to the root of your project, and run the following: +You can add the Dockerfile to the root of your project, and run the following to +build the container under a specified tag. ```sh $ docker build -t example-deployment . -$ docker run -p 7000:7000 example-deployment ``` -Once complete, open your browser at `http://localhost:7000` if running locally. +To run the image locally you can run: + +```sh +$ docker run -p -it 7000:7000 example-deployment +``` + +You should then start to get logs in your terminal, and then you can open your +browser at `http://localhost:7000` ## Heroku From fb118a6cf25c32a5a6e38f804a47299385fd3a59 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 26 Jan 2021 15:40:14 +0100 Subject: [PATCH 25/74] Add ADR for Plugin package Structure --- .../adr011-plugin-package-structure.md | 74 +++++++++++++++++++ microsite/sidebars.json | 3 +- mkdocs.yml | 1 + 3 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 docs/architecture-decisions/adr011-plugin-package-structure.md diff --git a/docs/architecture-decisions/adr011-plugin-package-structure.md b/docs/architecture-decisions/adr011-plugin-package-structure.md new file mode 100644 index 0000000000..9133e5b7c2 --- /dev/null +++ b/docs/architecture-decisions/adr011-plugin-package-structure.md @@ -0,0 +1,74 @@ +--- +id: adrs-adr011 +title: ADR011: Plugin Package Structure +description: Architecture Decision Record (ADR) for Plugin Package Structure +--- + +## Context + +A core feature of Backstage is the extensibility via plugins. The Backstage +repository is open for contributions of plugins. Even most of the core features +are implemented as plugins. A plugin consists of one or multiple packages in the +`plugins/` directory. Up till now, we have a simple conventions for naming +plugin packages: Plugins are named `x`, with the option of having a related +backend plugin called `x-backend` (where `x` is the plugin name, like `catalog` +or `techdocs`). There is a need for sharing code between the frontend and +backend of a plugin, between backend plugins, or components and hooks between +different frontend plugins +([some examples](https://github.com/backstage/backstage/issues/3655#issuecomment-758166746)). +This results in emerging plugin packages with shared code, like +`packages/catalog-client` or `packages/techdocs-common`. + +> There is a common phrase in software development: +> [Naming things is hard](https://martinfowler.com/bliki/TwoHardThings.html) + +To keep the contributed plugins consistent, this Architecture Decision Record +provides rules for naming plugin packages. + +## Decision + +We will place all plugin related code in the `plugins/` directory. The +`packages/` directory is reserved for core package of Backstage. + +We follow this structure for plugin packages (where `x` is the plugin name, for +example `catalog` or `techdocs`): + +- `x`: Contains the main frontend code of the plugin. +- `x-backend`: Contains the main backend code of the plugin. +- `x-react`: Contains shared widgets, hooks and similar that both the plugin + itself (`x`) and third-party frontend plugins can depend on. +- `x-node`: Contains utilities for backends that both the plugin backend itself + (`x-backend`) and third-party backend plugins can depend on. +- `x-common`: An isomorphic package with platform agnostic models, clients, and + utilities that all packages above or any third-party plugin package can depend + on. + +We prefix the package names with `@backstage/plugin-`. + +This structure is based on a +[suggestion in issue #3655](https://github.com/backstage/backstage/issues/3655#issuecomment-758166746). + +## Consequences + +We will actively migrate existing packages that are part of a plugin to the +`plugins/` folder. This affects packages like: + +- `packages/techdocs-common` which should be moved to `plugins/techdocs-node` + and named `@backstage/plugin-techdocs-node`. +- `packages/catalog-client` which will be part of a future + `plugins/catalog-common` and named `@backstage/plugin-catalog-common`. +- While the new location of `packages/catalog-model` should be + `plugins/catalog-common` we might want to do an exception here, as it's a very + central package. + +The limited set of rules might not be sufficient in the future. If additional +packages are required, we will revisit this decision and extend the pattern. + +If possible, we will add tools, such as lint rules, to help enforce the package +names and dependencies between them or CLI commands to generate these packages. + +The distinction between core packages and plugins helps us to setup +[CODEOWNERS](https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners) +in the repository. We can set the code owners for the `packages/` folder to the +core team and create additional rules (like `plugins/x*`) for plugin +maintainers. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index c2ccea8740..eb2d503476 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -184,7 +184,8 @@ "architecture-decisions/adrs-adr007", "architecture-decisions/adrs-adr008", "architecture-decisions/adrs-adr009", - "architecture-decisions/adrs-adr010" + "architecture-decisions/adrs-adr010", + "architecture-decisions/adrs-adr011" ], "Contribute": ["../CONTRIBUTING"], "Support": ["support/support", "support/project-structure"], diff --git a/mkdocs.yml b/mkdocs.yml index a2cbf11b4c..6236d1c1a0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -119,6 +119,7 @@ nav: - ADR008 - Default Catalog File Name: 'architecture-decisions/adr008-default-catalog-file-name.md' - ADR009 - Entity References: 'architecture-decisions/adr009-entity-references.md' - ADR010 - Luxon Date Library: 'architecture-decisions/adr010-luxon-date-library.md' + - ADR011 - Plugin Package Structure: 'architecture-decisions/adr011-plugin-package-structure.md' - Contribute: '../CONTRIBUTING.md' - Support: - 'support/support.md' From f8b652614e0965a6dd5fd90bca6e66a7e05c0297 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 10:50:12 -0800 Subject: [PATCH 26/74] fix: change issuer to iss to conform to JWT spec. Make issuer verification optional --- plugins/auth-backend/src/providers/aws-alb/provider.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/plugins/auth-backend/src/providers/aws-alb/provider.ts b/plugins/auth-backend/src/providers/aws-alb/provider.ts index 41dc5cf916..a5db2869b5 100644 --- a/plugins/auth-backend/src/providers/aws-alb/provider.ts +++ b/plugins/auth-backend/src/providers/aws-alb/provider.ts @@ -34,7 +34,7 @@ const ALB_JWT_HEADER = 'x-amzn-oidc-data'; */ type AwsAlbAuthProviderOptions = { region: string; - issuer: string; + issuer?: string; identityResolutionCallback: ExperimentalIdentityResolver; }; export const getJWTHeaders = (input: string) => { @@ -70,10 +70,7 @@ export class AwsAlbAuthProvider implements AuthProviderRouteHandlers { const key = await this.getKey(headers.kid); const payload = JWT.verify(jwt, key); - if ( - this.options.issuer !== '' && - headers.issuer !== this.options.issuer - ) { + if (this.options.issuer && headers.iss !== this.options.issuer) { throw new Error('issuer mismatch on JWT'); } @@ -116,7 +113,7 @@ export const createAwsAlbProvider = ({ identityResolver, }: AuthProviderFactoryOptions) => { const region = config.getString('region'); - const issuer = config.getString('iss'); + const issuer = config.getOptionalString('iss'); if (identityResolver !== undefined) { return new AwsAlbAuthProvider(logger, catalogApi, { region, From 4eaa060572b7c5285dfa426c201fc03e3760a347 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 10:54:14 -0800 Subject: [PATCH 27/74] Fix test, add changeset for ALB provider bug fix --- .changeset/stale-tools-confess.md | 5 +++++ plugins/auth-backend/src/providers/aws-alb/provider.test.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/stale-tools-confess.md diff --git a/.changeset/stale-tools-confess.md b/.changeset/stale-tools-confess.md new file mode 100644 index 0000000000..f5e6a6da02 --- /dev/null +++ b/.changeset/stale-tools-confess.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Fix issuer check diff --git a/plugins/auth-backend/src/providers/aws-alb/provider.test.ts b/plugins/auth-backend/src/providers/aws-alb/provider.test.ts index bae809971e..d05b6bbfaf 100644 --- a/plugins/auth-backend/src/providers/aws-alb/provider.test.ts +++ b/plugins/auth-backend/src/providers/aws-alb/provider.test.ts @@ -80,7 +80,7 @@ describe('AwsALBAuthProvider', () => { const mockResponseSend = jest.fn(); const mockRequest = ({ header: jest.fn(() => { - return 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImZvbyIsImlzc3VlciI6ImZvbyJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.zUkMYAuMwC1T0tyHMpxXrkbFDa4aGhB8d9um_tI2hsI'; + return 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6ImZvbyIsImlzcyI6ImZvbyJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.T2BNS4G-6RoiFnXc8Q8TiwdWzTpNitY8jcsGM3N3-Yo'; }), } as unknown) as express.Request; const mockRequestWithoutJwt = ({ From 5ad2d8b3be154037538244b61030d3230962bdfc Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 20:14:27 +0100 Subject: [PATCH 28/74] chore: updating config for tugboat updating --- .tugboat/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 20f716785c..065cc4d526 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -8,5 +8,9 @@ services: - yarn install - yarn tsc - yarn build + update: + - yarn install + - yarn tsc + - yarn build start: - yarn start-backend & From 99e53017d03b00fa1d10e4eabdb0e2e55df1df95 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 20:19:30 +0100 Subject: [PATCH 29/74] chore: reworking some more steps to see if this is how you build --- .tugboat/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 065cc4d526..4e393f2a6d 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -4,13 +4,13 @@ services: expose: 7000 default: true commands: - build: + init: - yarn install + build: - yarn tsc - yarn build update: - yarn install - - yarn tsc - - yarn build + - env start: - - yarn start-backend & + - node packages/backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From da8b9ef1d8d2c961e1a50cc181028f24c96499c0 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 11:41:41 -0800 Subject: [PATCH 30/74] add fields to config type for aws-alb provider --- plugins/auth-backend/config.d.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/auth-backend/config.d.ts b/plugins/auth-backend/config.d.ts index c748090711..84f71b52b9 100644 --- a/plugins/auth-backend/config.d.ts +++ b/plugins/auth-backend/config.d.ts @@ -72,6 +72,10 @@ export interface Config { onelogin?: { development: { [key: string]: string }; }; + awsalb?: { + issuer?: string; + region: string; + }; }; }; } From 59f9a5baad6294496af40b361528f5b98660da41 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 20:44:57 +0100 Subject: [PATCH 31/74] feat: add in the tugboat config --- .tugboat/tugboat.app-config.production.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .tugboat/tugboat.app-config.production.yaml diff --git a/.tugboat/tugboat.app-config.production.yaml b/.tugboat/tugboat.app-config.production.yaml new file mode 100644 index 0000000000..f606574d89 --- /dev/null +++ b/.tugboat/tugboat.app-config.production.yaml @@ -0,0 +1,13 @@ +app: + title: Backstage Tugboat Preview + baseUrl: + $env: TUGBOAT_DEFAULT_SERVICE_URL + +backend: + baseUrl: + $env: TUGBOAT_DEFAULT_SERVICE_URL + cors: + origin: + $env: TUGBOAT_DEFAULT_SERVICE_URL + methods: [GET, POST, PUT, DELETE] + credentials: true From e877faa6eb5595605e5524f11168ce3926063099 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 20:53:23 +0100 Subject: [PATCH 32/74] chore: fixing config --- .tugboat/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 4e393f2a6d..aaca7f6d86 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -11,6 +11,7 @@ services: - yarn build update: - yarn install - - env + - yarn tsc + - yarn build start: - node packages/backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From e20b984a374ecef1de0dd3349f77c5bd67f7a322 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 26 Jan 2021 21:37:49 +0100 Subject: [PATCH 33/74] chore: updating base image --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index aaca7f6d86..0c8c0329e3 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -1,6 +1,6 @@ services: backstage: - image: tugboatqa/node:lts + image: node:lts-alpine expose: 7000 default: true commands: From f50d284fb8373cf59f0de59d40e9c1b99089e7e1 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 13:27:59 -0800 Subject: [PATCH 34/74] Update .changeset/stale-tools-confess.md Co-authored-by: Patrik Oldsberg --- .changeset/stale-tools-confess.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/stale-tools-confess.md b/.changeset/stale-tools-confess.md index f5e6a6da02..10f2203010 100644 --- a/.changeset/stale-tools-confess.md +++ b/.changeset/stale-tools-confess.md @@ -2,4 +2,4 @@ '@backstage/plugin-auth-backend': patch --- -Fix issuer check +Fix AWS ALB issuer check From a7c0da02e8d8515e47880e12d6bbc73bbdb966f8 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Sun, 17 Jan 2021 18:16:19 -0500 Subject: [PATCH 35/74] pass in the specified git branch ref when cloning a single branch --- packages/backend-common/src/scm/git.ts | 3 ++- .../scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index b0a1df6541..55a8746d6e 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -86,13 +86,14 @@ export class Git { return git.commit({ fs, dir, message, author, committer }); } - async clone({ url, dir }: { url: string; dir: string }): Promise { + async clone({ url, dir, ref }: { url: string; dir: string; ref?: string }): Promise { this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`); return git.clone({ fs, http, url, dir, + ref, singleBranch: true, depth: 1, onProgress: this.onProgressHandler(), diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index 1fca2ff0f9..7824687155 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -42,6 +42,7 @@ export class GitlabPreparer implements PreparerBase { const parsedGitLocation = parseGitUrl(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); + const ref = parsedGitLocation.toString('ref'); const tempDir = await fs.promises.mkdtemp( path.join(workingDirectory, templateId), ); @@ -61,6 +62,7 @@ export class GitlabPreparer implements PreparerBase { await git.clone({ url: repositoryCheckoutUrl, + ref: ref, dir: tempDir, }); From 97df886e1c2ca1e2ba55bf1e6f04c2d4e8c16192 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Tue, 19 Jan 2021 10:32:15 -0500 Subject: [PATCH 36/74] KISS: no need for toString() --- .../scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index 7824687155..0ce5466915 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -42,7 +42,7 @@ export class GitlabPreparer implements PreparerBase { const parsedGitLocation = parseGitUrl(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); - const ref = parsedGitLocation.toString('ref'); + const ref = parsedGitLocation.ref; const tempDir = await fs.promises.mkdtemp( path.join(workingDirectory, templateId), ); From 26a3a6cf030d3d7e926a0b23d2adb5ef9cd4e2a6 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Thu, 21 Jan 2021 15:37:49 -0500 Subject: [PATCH 37/74] bump patch-level versions --- .changeset/breezy-meals-lie.md | 6 ++++++ packages/backend-common/package.json | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 .changeset/breezy-meals-lie.md diff --git a/.changeset/breezy-meals-lie.md b/.changeset/breezy-meals-lie.md new file mode 100644 index 0000000000..d9bc3f3d22 --- /dev/null +++ b/.changeset/breezy-meals-lie.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +include ref when cloning from GitLab diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c698351615..350b7bd8f8 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.5.0", + "version": "0.5.1", "main": "src/index.ts", "types": "src/index.ts", "private": false, From 85cdcb2c6b6fb9ccfa85fef2945c0bafbb827a3f Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Thu, 21 Jan 2021 15:47:56 -0500 Subject: [PATCH 38/74] Prettier --- packages/backend-common/src/scm/git.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-common/src/scm/git.ts b/packages/backend-common/src/scm/git.ts index 55a8746d6e..5b24d23b99 100644 --- a/packages/backend-common/src/scm/git.ts +++ b/packages/backend-common/src/scm/git.ts @@ -86,7 +86,15 @@ export class Git { return git.commit({ fs, dir, message, author, committer }); } - async clone({ url, dir, ref }: { url: string; dir: string; ref?: string }): Promise { + async clone({ + url, + dir, + ref, + }: { + url: string; + dir: string; + ref?: string; + }): Promise { this.config.logger?.info(`Cloning repo {dir=${dir},url=${url}}`); return git.clone({ fs, From c7de259c929c6feab8cf1fef1c76e7b3b8e6f5d4 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Thu, 21 Jan 2021 16:14:09 -0500 Subject: [PATCH 39/74] add ref parameter to clone --- .../src/scaffolder/stages/prepare/gitlab.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index 19cbf793e4..8ea1d77859 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -89,6 +89,7 @@ describe('GitLabPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', dir: expect.any(String), + ref: expect.any(String), }); }); @@ -113,6 +114,7 @@ describe('GitLabPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', dir: expect.any(String), + ref: expect.any(String), }); }); From 9ba95e5085a8ee5b76fdc01387ea74223d11b23b Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Thu, 21 Jan 2021 16:40:21 -0500 Subject: [PATCH 40/74] undo manual package version bump done in error --- packages/backend-common/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 350b7bd8f8..c698351615 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.5.1", + "version": "0.5.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index cd7dab3444..914b151eeb 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.5.1", + "version": "0.5.0", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", From ea21f9c12b083af45b29a88c35ce3dc4cc115fe9 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Fri, 22 Jan 2021 13:18:21 -0500 Subject: [PATCH 41/74] pass in the specified git branch ref when cloning a single branch --- .../src/scaffolder/stages/prepare/github.test.ts | 2 ++ .../scaffolder-backend/src/scaffolder/stages/prepare/github.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index b43545f5fc..4051e5d30d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -89,6 +89,7 @@ describe('GitHubPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://github.com/benjdlambert/backstage-graphql-template', dir: expect.any(String), + ref: expect.any(String), }); }); @@ -100,6 +101,7 @@ describe('GitHubPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://github.com/benjdlambert/backstage-graphql-template', dir: expect.any(String), + ref: expect.any(String), }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts index cfeff454e7..02cbb1c5cd 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.ts @@ -42,6 +42,7 @@ export class GithubPreparer implements PreparerBase { const parsedGitLocation = parseGitUrl(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); + const ref = parsedGitLocation.ref; const tempDir = await fs.promises.mkdtemp( path.join(workingDirectory, templateId), ); @@ -63,6 +64,7 @@ export class GithubPreparer implements PreparerBase { await git.clone({ url: repositoryCheckoutUrl, + ref: ref, dir: tempDir, }); From 00d9d6ae454f61ff16335b65265ffb5ae331b7db Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Fri, 22 Jan 2021 15:01:01 -0500 Subject: [PATCH 42/74] Add missing ref fix that was applied to github and gitlab. --- .../src/scaffolder/stages/prepare/bitbucket.test.ts | 2 ++ .../src/scaffolder/stages/prepare/bitbucket.ts | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts index ff74fffdd7..32822ae289 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.test.ts @@ -89,6 +89,7 @@ describe('BitbucketPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://bitbucket.org/backstage-project/backstage-repo', dir: expect.any(String), + ref: expect.any(String), }); }); @@ -113,6 +114,7 @@ describe('BitbucketPreparer', () => { expect(mockGitClient.clone).toHaveBeenCalledWith({ url: 'https://bitbucket.org/backstage-project/backstage-repo', dir: expect.any(String), + ref: expect.any(String), }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts index 5020a5f658..04f48e3005 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/bitbucket.ts @@ -49,15 +49,15 @@ export class BitbucketPreparer implements PreparerBase { const logger = opts.logger; const templateId = template.metadata.name; - const repo = parseGitUrl(location); - const repositoryCheckoutUrl = repo.toString('https'); - + const parsedGitLocation = parseGitUrl(location); + const repositoryCheckoutUrl = parsedGitLocation.toString('https'); + const ref = parsedGitLocation.ref; const tempDir = await fs.promises.mkdtemp( path.join(workingDirectory, templateId), ); const templateDirectory = path.join( - `${path.dirname(repo.filepath)}`, + `${path.dirname(parsedGitLocation.filepath)}`, template.spec.path ?? '.', ); @@ -73,6 +73,7 @@ export class BitbucketPreparer implements PreparerBase { await git.clone({ url: repositoryCheckoutUrl, + ref: ref, dir: tempDir, }); From b34d2514d716c010d4ea0bcb8d9ade2e19677f57 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Fri, 22 Jan 2021 15:29:20 -0500 Subject: [PATCH 43/74] improve the changset description --- .changeset/breezy-meals-lie.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.changeset/breezy-meals-lie.md b/.changeset/breezy-meals-lie.md index d9bc3f3d22..cbdd876614 100644 --- a/.changeset/breezy-meals-lie.md +++ b/.changeset/breezy-meals-lie.md @@ -3,4 +3,16 @@ '@backstage/plugin-scaffolder-backend': patch --- -include ref when cloning from GitLab +Honor the branch ref in the url when cloning. + +This fixes a bug in the scaffolder prepare stage where a non-default branch +was specified in the scaffolder URL but the default branch was cloned. +For example, even though the `other` branch is specified in this example, the +`master` branch was actually cloned: + +```yaml +catalog: + locations: + - type: url + target: https://github.com/backstage/backstage/blob/other/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml +``` From c05baec7fdcd599bc72d7d8023cbbbede8634d9c Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Sat, 23 Jan 2021 14:12:18 -0500 Subject: [PATCH 44/74] Fix gitlab prepare fails with a 404. A GitLab URL not ending in .git requires a redirect which isomorphic-git doesn't follow, resulting in a 404. This incorporates @ruloweb's fix in #4134. --- .changeset/breezy-meals-lie.md | 2 ++ .../src/scaffolder/stages/prepare/gitlab.test.ts | 4 ++-- .../src/scaffolder/stages/prepare/gitlab.ts | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.changeset/breezy-meals-lie.md b/.changeset/breezy-meals-lie.md index cbdd876614..4ca4f1ac35 100644 --- a/.changeset/breezy-meals-lie.md +++ b/.changeset/breezy-meals-lie.md @@ -16,3 +16,5 @@ catalog: - type: url target: https://github.com/backstage/backstage/blob/other/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml ``` + +This also fixes a 404 in the prepare stage for GitLab URLs. diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts index 8ea1d77859..0b77fd813b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.test.ts @@ -87,7 +87,7 @@ describe('GitLabPreparer', () => { await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', + url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git', dir: expect.any(String), ref: expect.any(String), }); @@ -112,7 +112,7 @@ describe('GitLabPreparer', () => { await preparer.prepare(mockEntity, { logger: getVoidLogger() }); expect(mockGitClient.clone).toHaveBeenCalledWith({ - url: 'https://gitlab.com/benjdlambert/backstage-graphql-template', + url: 'https://gitlab.com/benjdlambert/backstage-graphql-template.git', dir: expect.any(String), ref: expect.any(String), }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts index 0ce5466915..4368c1c242 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/gitlab.ts @@ -41,6 +41,7 @@ export class GitlabPreparer implements PreparerBase { const templateId = template.metadata.name; const parsedGitLocation = parseGitUrl(location); + parsedGitLocation.git_suffix = true; const repositoryCheckoutUrl = parsedGitLocation.toString('https'); const ref = parsedGitLocation.ref; const tempDir = await fs.promises.mkdtemp( From 119c6386fda3e4262dadf60355749f4ceadbd4d5 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Sat, 23 Jan 2021 17:21:39 -0500 Subject: [PATCH 45/74] Revert "undo manual package version bump done in error" This reverts commit 44c5ec6feb5109ae97c1a66b885da3c6440de0d5. --- packages/backend-common/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c698351615..350b7bd8f8 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.5.0", + "version": "0.5.1", "main": "src/index.ts", "types": "src/index.ts", "private": false, diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 914b151eeb..cd7dab3444 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-scaffolder-backend", - "version": "0.5.0", + "version": "0.5.1", "main": "src/index.ts", "types": "src/index.ts", "license": "Apache-2.0", From aae1dca6c6f4da6011c0999fa4e0743fa6838468 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Sat, 23 Jan 2021 17:23:47 -0500 Subject: [PATCH 46/74] Revert "bump patch-level versions" This reverts commit 96c99d75ae6523c049e6f51a3f72e7d47591a324. --- packages/backend-common/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 350b7bd8f8..c698351615 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -1,7 +1,7 @@ { "name": "@backstage/backend-common", "description": "Common functionality library for Backstage backends", - "version": "0.5.1", + "version": "0.5.0", "main": "src/index.ts", "types": "src/index.ts", "private": false, From ea59a2b363b32fe42d1d096608ceea90dc827c0e Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Sat, 23 Jan 2021 17:42:26 -0500 Subject: [PATCH 47/74] Azure branch support --- .../src/scaffolder/stages/prepare/azure.test.ts | 4 +++- .../scaffolder-backend/src/scaffolder/stages/prepare/azure.ts | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts index 8dd72c35e9..89ca8e2f8f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.test.ts @@ -45,7 +45,7 @@ describe('AzurePreparer', () => { metadata: { annotations: { [LOCATION_ANNOTATION]: - 'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml', + 'url:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml&version=GBmaster', }, name: 'graphql-starter', title: 'GraphQL Service', @@ -112,6 +112,7 @@ describe('AzurePreparer', () => { url: 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', dir: expect.any(String), + ref: 'master', }); }); @@ -124,6 +125,7 @@ describe('AzurePreparer', () => { url: 'https://dev.azure.com/backstage-org/backstage-project/_git/template-repo', dir: expect.any(String), + ref: 'master', }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts index 8b204fa799..67f089965d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/azure.ts @@ -42,6 +42,7 @@ export class AzurePreparer implements PreparerBase { const parsedGitLocation = parseGitUrl(location); const repositoryCheckoutUrl = parsedGitLocation.toString('https'); + const ref = parsedGitLocation.ref; const tempDir = await fs.promises.mkdtemp( path.join(workingDirectory, templateId), ); @@ -63,6 +64,7 @@ export class AzurePreparer implements PreparerBase { await git.clone({ url: repositoryCheckoutUrl, + ref: ref, dir: tempDir, }); From 6ddfdca12364838218ce8b7a1e479ca63af95a04 Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 16:54:52 -0800 Subject: [PATCH 48/74] fix aws alb auth provider key caching --- plugins/auth-backend/src/providers/aws-alb/provider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/auth-backend/src/providers/aws-alb/provider.ts b/plugins/auth-backend/src/providers/aws-alb/provider.ts index a5db2869b5..61ea10947e 100644 --- a/plugins/auth-backend/src/providers/aws-alb/provider.ts +++ b/plugins/auth-backend/src/providers/aws-alb/provider.ts @@ -95,13 +95,13 @@ export class AwsAlbAuthProvider implements AuthProviderRouteHandlers { async getKey(keyId: string): Promise { const optionalCacheKey = this.keyCache.get(keyId); if (optionalCacheKey) { - return optionalCacheKey; + return crypto.createPublicKey(optionalCacheKey); } const keyText: string = await fetch( `https://public-keys.auth.elb.${this.options.region}.amazonaws.com/${keyId}`, ).then(response => response.text()); const keyValue = crypto.createPublicKey(keyText); - this.keyCache.set(keyId, keyValue); + this.keyCache.set(keyId, keyValue.export({ format: 'pem', type: 'spki' })); return keyValue; } } From d7b1d317f623734790ddcf7e333c12753d43d97d Mon Sep 17 00:00:00 2001 From: Jonah Back Date: Tue, 26 Jan 2021 16:59:40 -0800 Subject: [PATCH 49/74] Add changeset for alb aws provider fix --- .changeset/fuzzy-points-whisper.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fuzzy-points-whisper.md diff --git a/.changeset/fuzzy-points-whisper.md b/.changeset/fuzzy-points-whisper.md new file mode 100644 index 0000000000..d1af2b2dcf --- /dev/null +++ b/.changeset/fuzzy-points-whisper.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': patch +--- + +Fixed serialization issue with caching of public keys in AWS ALB auth provider From adee7b3482b1d4d496ac6f59c6b53f76ccfb0c4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jan 2021 05:01:17 +0000 Subject: [PATCH 50/74] chore(deps-dev): bump @storybook/react from 6.1.14 to 6.1.15 Bumps [@storybook/react](https://github.com/storybookjs/storybook/tree/HEAD/app/react) from 6.1.14 to 6.1.15. - [Release notes](https://github.com/storybookjs/storybook/releases) - [Changelog](https://github.com/storybookjs/storybook/blob/next/CHANGELOG.md) - [Commits](https://github.com/storybookjs/storybook/commits/v6.1.15/app/react) Signed-off-by: dependabot[bot] --- yarn.lock | 211 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 109 insertions(+), 102 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7c743bc5b3..36a133fd37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5301,17 +5301,17 @@ global "^4.3.2" regenerator-runtime "^0.13.7" -"@storybook/addons@6.1.14", "@storybook/addons@^6.1.11": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.1.14.tgz#2b81304bbe696923df95cdcf85cfc592d10f4065" - integrity sha512-HlpmV7aejp/MeW8bo/WKME3i71gi0men9qcwoovjDjnSF6jXoNLT336a5udKXdHqYSZgzdyURlgLtilCWkWaJQ== +"@storybook/addons@6.1.15", "@storybook/addons@^6.1.11": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/addons/-/addons-6.1.15.tgz#09eb8d962f58bd20b4ac2f83b515831c83226352" + integrity sha512-ENyHapLFOG93VaoQXPX8O3IWjLRyVBox9C9P20LMruKX/SfXAXx20qsoAWKKPGssopyOin17aoQX9pj+lFmCZQ== dependencies: - "@storybook/api" "6.1.14" - "@storybook/channels" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/core-events" "6.1.14" - "@storybook/router" "6.1.14" - "@storybook/theming" "6.1.14" + "@storybook/api" "6.1.15" + "@storybook/channels" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/core-events" "6.1.15" + "@storybook/router" "6.1.15" + "@storybook/theming" "6.1.15" core-js "^3.0.1" global "^4.3.2" regenerator-runtime "^0.13.7" @@ -5341,20 +5341,20 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" -"@storybook/api@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/api/-/api-6.1.14.tgz#20035dd336aba1c5a0f8c83c8c14a2edaf4db891" - integrity sha512-gWcC/xEW8HL5DsocLujHBUdoNsl4YW1Zx1Y4SBbLCyrhj8v4JudJpylwJpOUBDe/GESXq1zqvNKvUPtI8DQNyw== +"@storybook/api@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/api/-/api-6.1.15.tgz#285ba42f7a8efcd3bd0e586a5e978487d826fbb4" + integrity sha512-C4D08e2ZbSe62nNKtmh9YBraoWb2j6Chw8VCkuj91kuKHh3YDNc1gjj5Fi+KYZwIcy0EllzW3RFQs+YR1/Vg1g== dependencies: "@reach/router" "^1.3.3" - "@storybook/channels" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/core-events" "6.1.14" + "@storybook/channels" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/core-events" "6.1.15" "@storybook/csf" "0.0.1" - "@storybook/router" "6.1.14" + "@storybook/router" "6.1.15" "@storybook/semver" "^7.3.2" - "@storybook/theming" "6.1.14" - "@types/reach__router" "^1.3.5" + "@storybook/theming" "6.1.15" + "@types/reach__router" "^1.3.7" core-js "^3.0.1" fast-deep-equal "^3.1.1" global "^4.3.2" @@ -5379,14 +5379,14 @@ qs "^6.6.0" telejson "^5.0.2" -"@storybook/channel-postmessage@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/channel-postmessage/-/channel-postmessage-6.1.14.tgz#41f3115895010dad9fb30f4ac381e4f904b1e50c" - integrity sha512-If83dXXW9mKIRuvuWhWa/zkEw/F0FDgikp33x8436J3rWCh3recp27kffFRrKG0YDMpFSk/Ci5G47E9zn9SCjw== +"@storybook/channel-postmessage@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/channel-postmessage/-/channel-postmessage-6.1.15.tgz#80ea2346d18496f9710dd7f87fd2a9eca46ef36f" + integrity sha512-Es4B5zpLrW28KSbY8FhGVEDgUnKspJ7wPuJyKExUpZ5L9w52RkTD6lRnVPzLUfoQ4luPsExy5fiuo878/Wc9ag== dependencies: - "@storybook/channels" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/core-events" "6.1.14" + "@storybook/channels" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/core-events" "6.1.15" core-js "^3.0.1" global "^4.3.2" qs "^6.6.0" @@ -5401,10 +5401,10 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" -"@storybook/channels@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/channels/-/channels-6.1.14.tgz#c479190ebb853a603f3ed90fc470534a02eb46eb" - integrity sha512-vP19IB2FXj8SiFbQ9ETljEBienL+KRMLgMzz3Ta3nZj/OfjJJbIuj42ZfexQGV4mS0Bo+OW+qT7VMIY6fulnFw== +"@storybook/channels@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/channels/-/channels-6.1.15.tgz#22bb06a671a5ae09d2537bcf63aaf90d7f6b9f6b" + integrity sha512-HIKHDeL/0BDk9a7xc2PLiFFoHjUMKUd2djhUGdeKgdKqoWejp4JJ60fI68+2QuSRbkB8k+rAwmuWJzV7EfB5fg== dependencies: core-js "^3.0.1" ts-dedent "^2.0.0" @@ -5434,16 +5434,16 @@ ts-dedent "^2.0.0" util-deprecate "^1.0.2" -"@storybook/client-api@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/client-api/-/client-api-6.1.14.tgz#6daf56743cc72e13f05fff3d2ac554897cc9f9fd" - integrity sha512-pIDSlS48bhJdtgNg7sXV1NmLJtB0ebRHJI9htIiqtL7EGQenb4+Bbwflhj1j51OEkuM+bQsAAZxq5deiUQEGVw== +"@storybook/client-api@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/client-api/-/client-api-6.1.15.tgz#8f8ead111459b94621571bdb2276f8a0aace17b1" + integrity sha512-iwuDlgNdB6Y4OidlhWPob3tEIax9taymdKEe9by4rLJ3nfXu7viHcvCAjN24oI4NFW3NZsmtqJotgftRYk0r1Q== dependencies: - "@storybook/addons" "6.1.14" - "@storybook/channel-postmessage" "6.1.14" - "@storybook/channels" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/core-events" "6.1.14" + "@storybook/addons" "6.1.15" + "@storybook/channel-postmessage" "6.1.15" + "@storybook/channels" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/core-events" "6.1.15" "@storybook/csf" "0.0.1" "@types/qs" "^6.9.0" "@types/webpack-env" "^1.15.3" @@ -5466,10 +5466,10 @@ core-js "^3.0.1" global "^4.3.2" -"@storybook/client-logger@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-6.1.14.tgz#216b9c1332ffa3a3473dad837780a3b14f686bae" - integrity sha512-NSO8nVsp6o0eoQ1Drlu66KXpl6DPuq02Kj8AhttGzvqSYB50SV4CV+wceBcg77tIVu5QmQ+71hAEVXhx7sjRHA== +"@storybook/client-logger@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/client-logger/-/client-logger-6.1.15.tgz#b558d6ecbee82c038d684717d8c598eaa4a9324d" + integrity sha512-lUpatG8SxzrUapWMsIPWiR+5qRVT5ebn8tGHQeBeRHXbdmEqyq5DOlrotLUemkA5nNTCs1pMFNvKSpCHznG+fg== dependencies: core-js "^3.0.1" global "^4.3.2" @@ -5500,15 +5500,15 @@ react-textarea-autosize "^8.1.1" ts-dedent "^2.0.0" -"@storybook/components@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/components/-/components-6.1.14.tgz#4ea47edfa0a3e4a26882aa5a1eb90c1ec86e6f71" - integrity sha512-Nxsp/9o1tqfY8s6RBWNHyM03A5D9k56Kr/4VNa++CbDrz1+TIxpYlDgS4sllUlXyvICLfk3sUtg3KS5CPl2iZA== +"@storybook/components@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/components/-/components-6.1.15.tgz#b4a2af23ee6b9cba4c255191eae3d3463e29bfb7" + integrity sha512-lPbA/zyBfctdlpDhRTcRFLWlZPJ3PB4+wI0FUvYs69iG3/bNbQPYu8vRmNhCZOsaGt+b+dik4Tfcth8Bu+eQug== dependencies: "@popperjs/core" "^2.5.4" - "@storybook/client-logger" "6.1.14" + "@storybook/client-logger" "6.1.15" "@storybook/csf" "0.0.1" - "@storybook/theming" "6.1.14" + "@storybook/theming" "6.1.15" "@types/overlayscrollbars" "^1.9.0" "@types/react-color" "^3.0.1" "@types/react-syntax-highlighter" "11.0.4" @@ -5533,17 +5533,17 @@ dependencies: core-js "^3.0.1" -"@storybook/core-events@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/core-events/-/core-events-6.1.14.tgz#a3165e32cefd6be7326bbad4b8140653bdfa0426" - integrity sha512-tpM3VDvzqgRY7S17CRglgt1625rxNoyEwrLQiNcZkUPyO0rpaacPqVEbPCtcTmUeboI1bLdnSQIjT9B0/Y2Pww== +"@storybook/core-events@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/core-events/-/core-events-6.1.15.tgz#f66e30cbed8afdb8df2254d2aa47fe139e641c60" + integrity sha512-2sz02hdGZshanoq83jaB+goAcapVEWrxe+RJZn/gu2OymlEioWNjPPtOVGgi5DNIiJFnYvc66adayNwX39+tDA== dependencies: core-js "^3.0.1" -"@storybook/core@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/core/-/core-6.1.14.tgz#17e724a5b94d6e1bb557e213b8176660d2d14762" - integrity sha512-lHKZmfLAo2VGtF/yrZkkWMYgmFRNKbzIDxYJGp8USyUQyTfEpz2qqJlBdoD6rxr1hFPM2954tIKwh8iPhT2PFQ== +"@storybook/core@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/core/-/core-6.1.15.tgz#7ff8c314d3857497bf2e26c69a1fa93ef37301aa" + integrity sha512-mQeKAXcowUwF+pOdWZEFwb5M6sz4yv5cOv1vTci3/1pMmB8QpYlH+P61p4lsRO17Vlak70h18TworPka/4+mhA== dependencies: "@babel/core" "^7.12.3" "@babel/plugin-proposal-class-properties" "^7.12.1" @@ -5567,20 +5567,20 @@ "@babel/preset-react" "^7.12.1" "@babel/preset-typescript" "^7.12.1" "@babel/register" "^7.12.1" - "@storybook/addons" "6.1.14" - "@storybook/api" "6.1.14" - "@storybook/channel-postmessage" "6.1.14" - "@storybook/channels" "6.1.14" - "@storybook/client-api" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/components" "6.1.14" - "@storybook/core-events" "6.1.14" + "@storybook/addons" "6.1.15" + "@storybook/api" "6.1.15" + "@storybook/channel-postmessage" "6.1.15" + "@storybook/channels" "6.1.15" + "@storybook/client-api" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/components" "6.1.15" + "@storybook/core-events" "6.1.15" "@storybook/csf" "0.0.1" - "@storybook/node-logger" "6.1.14" - "@storybook/router" "6.1.14" + "@storybook/node-logger" "6.1.15" + "@storybook/router" "6.1.15" "@storybook/semver" "^7.3.2" - "@storybook/theming" "6.1.14" - "@storybook/ui" "6.1.14" + "@storybook/theming" "6.1.15" + "@storybook/ui" "6.1.15" "@types/glob-base" "^0.3.0" "@types/micromatch" "^4.0.1" "@types/node-fetch" "^2.5.4" @@ -5654,10 +5654,10 @@ dependencies: lodash "^4.17.15" -"@storybook/node-logger@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/node-logger/-/node-logger-6.1.14.tgz#e5294f986e3ec5c67b2738895b9d16c9a2b667fa" - integrity sha512-3jrw7coAwFXZu4qK1vm54bCPhNRvxjG+7jISbhhocDoNIv0nLWL3+tJyrC5/k/XHQiUlLkhEzpMaASADmkttNw== +"@storybook/node-logger@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/node-logger/-/node-logger-6.1.15.tgz#fcf786d3a323feb6821e40e26f98a513a60d1a79" + integrity sha512-lrO0ei3W7BRci2iUkWTr/rXgHkzxwZTrlkx0iBzbQQRy7K1AJ9bjzhurCH9B8C9XGLmn60LXT81RWD3iCLZjcw== dependencies: "@types/npmlog" "^4.1.2" chalk "^4.0.0" @@ -5666,16 +5666,16 @@ pretty-hrtime "^1.0.3" "@storybook/react@^6.1.11": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/react/-/react-6.1.14.tgz#436e9b90096b1d7c83f7f073b5baf47212b2e425" - integrity sha512-M99wHjc/5z+Wz1FdFaScVs6dyAi/6PdcIx5Fyip6Qd8aKwm1XyYoOMql5Vu3Cf560feDYCKS4phzyEZ7EJy+EQ== + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/react/-/react-6.1.15.tgz#e17d00b05b8980ad381ba701805309ed46d1fdcd" + integrity sha512-7WoYLOZuAlzgQsL9oy4JCr9NcB4NBCuxslPSncN5l/7ewGXgfVXTAOMOfw+EVNrtUeVJU2fC8gFiHVl0SJpTZw== dependencies: "@babel/preset-flow" "^7.12.1" "@babel/preset-react" "^7.12.1" "@pmmmwh/react-refresh-webpack-plugin" "^0.4.2" - "@storybook/addons" "6.1.14" - "@storybook/core" "6.1.14" - "@storybook/node-logger" "6.1.14" + "@storybook/addons" "6.1.15" + "@storybook/core" "6.1.15" + "@storybook/node-logger" "6.1.15" "@storybook/semver" "^7.3.2" "@types/webpack-env" "^1.15.3" babel-plugin-add-react-displayname "^0.0.5" @@ -5704,13 +5704,13 @@ memoizerific "^1.11.3" qs "^6.6.0" -"@storybook/router@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/router/-/router-6.1.14.tgz#f6aef8c9dabf19bf06dddd80907e66369261fdde" - integrity sha512-rMaUCYzgfVLwFWo3A1Q/weSv8FBqCLmHY+3+t6ao7OV6NYjR0XgLKRzHrXq1uYdbMxWeIKhN2tIt/LR43bmDjQ== +"@storybook/router@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/router/-/router-6.1.15.tgz#e0cd7440a2ddc9b265e506b1cb590d3eeab56476" + integrity sha512-HlxDkGpiTSxXCJuqRoZ9Viq6Y/h/7efI8LPhhopr50qWRBTh/PEQzDqWBXG3sj8ISmi9GyUaTSAuqRwdA3lJQQ== dependencies: "@reach/router" "^1.3.3" - "@types/reach__router" "^1.3.5" + "@types/reach__router" "^1.3.7" core-js "^3.0.1" global "^4.3.2" memoizerific "^1.11.3" @@ -5759,15 +5759,15 @@ resolve-from "^5.0.0" ts-dedent "^2.0.0" -"@storybook/theming@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/theming/-/theming-6.1.14.tgz#fecb66cab22d3b3218b4a98a9c210eb8a7be91e8" - integrity sha512-S+t30y4FqBTXWoVr+dtxVJ/ywiQGHBclBd9aUunbdCV4mMFra5InNo2CWn+RJlNEauLZ93gRIEzSFchIbzLk1A== +"@storybook/theming@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/theming/-/theming-6.1.15.tgz#01083ab89904dd959429b0b3fd1c76bd0ecc59ef" + integrity sha512-88IdYaPzp4NMKf/GKBrPggxD6/d/lkdQ4SNowXxN9g9eONd9M7HtTbjuJGRCbGMJ52xGcbpj2exEnAqKQ2iodA== dependencies: "@emotion/core" "^10.1.1" "@emotion/is-prop-valid" "^0.8.6" "@emotion/styled" "^10.0.23" - "@storybook/client-logger" "6.1.14" + "@storybook/client-logger" "6.1.15" core-js "^3.0.1" deep-object-diff "^1.1.0" emotion-theming "^10.0.19" @@ -5777,21 +5777,21 @@ resolve-from "^5.0.0" ts-dedent "^2.0.0" -"@storybook/ui@6.1.14": - version "6.1.14" - resolved "https://registry.npmjs.org/@storybook/ui/-/ui-6.1.14.tgz#766d696480ee6f6a5a0454ccb2f101c38a0eb9d2" - integrity sha512-DTW2TM05jTMKxh8LzUGk3g5a528PgJxrtgODFU6zzwSg2+LwdmSDtd1HAxopt2vpfTyQyX+6WN2H+lMNwfQTAQ== +"@storybook/ui@6.1.15": + version "6.1.15" + resolved "https://registry.npmjs.org/@storybook/ui/-/ui-6.1.15.tgz#a0f6c49fcf81cf172cd2de4c8dba2be1296891f6" + integrity sha512-quyhJWlOxhk95he7s5/TSYM3eEsaz3s4+98kUZE6r3ssME8u6zDvqa/qa6EWs5/nvZ2V3+12efIzCNbiiT3v3g== dependencies: "@emotion/core" "^10.1.1" - "@storybook/addons" "6.1.14" - "@storybook/api" "6.1.14" - "@storybook/channels" "6.1.14" - "@storybook/client-logger" "6.1.14" - "@storybook/components" "6.1.14" - "@storybook/core-events" "6.1.14" - "@storybook/router" "6.1.14" + "@storybook/addons" "6.1.15" + "@storybook/api" "6.1.15" + "@storybook/channels" "6.1.15" + "@storybook/client-logger" "6.1.15" + "@storybook/components" "6.1.15" + "@storybook/core-events" "6.1.15" + "@storybook/router" "6.1.15" "@storybook/semver" "^7.3.2" - "@storybook/theming" "6.1.14" + "@storybook/theming" "6.1.15" "@types/markdown-to-jsx" "^6.11.0" copy-to-clipboard "^3.0.8" core-js "^3.0.1" @@ -7093,6 +7093,13 @@ "@types/history" "*" "@types/react" "*" +"@types/reach__router@^1.3.7": + version "1.3.7" + resolved "https://registry.npmjs.org/@types/reach__router/-/reach__router-1.3.7.tgz#de8ab374259ae7f7499fc1373b9697a5f3cd6428" + integrity sha512-cyBEb8Ef3SJNH5NYEIDGPoMMmYUxROatuxbICusVRQIqZUB85UCt6R2Ok60tKS/TABJsJYaHyNTW3kqbpxlMjg== + dependencies: + "@types/react" "*" + "@types/react-color@^3.0.1": version "3.0.4" resolved "https://registry.npmjs.org/@types/react-color/-/react-color-3.0.4.tgz#c63daf012ad067ac0127bdd86725f079d02082bd" From 4f6d779b78195f6ad86f303cd6c3e79933344b47 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 09:42:50 +0100 Subject: [PATCH 51/74] chore: run the built main for now, --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 0c8c0329e3..af30015047 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -14,4 +14,4 @@ services: - yarn tsc - yarn build start: - - node packages/backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & + - node packages/backend/dist/main --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From f683dfbee5af261cd344022d75c30d119775fdbc Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 10:02:19 +0100 Subject: [PATCH 52/74] chore: fixing config again --- .tugboat/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index af30015047..d130f60554 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -1,6 +1,6 @@ services: backstage: - image: node:lts-alpine + image: tugboatqa/node:lts expose: 7000 default: true commands: @@ -14,4 +14,5 @@ services: - yarn tsc - yarn build start: - - node packages/backend/dist/main --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & + # This should be production, but we need to run backend:bundle like the dockerfile does. + - yarn start-backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From 233d519ca19ac180495a58a6a40d39e9aeebda09 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 10:23:53 +0100 Subject: [PATCH 53/74] chore: think this is what it's supposed to look like haha --- .tugboat/config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index d130f60554..bdef387f3f 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -8,11 +8,9 @@ services: - yarn install build: - yarn tsc - - yarn build + - yarn build --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml update: - yarn install - - yarn tsc - - yarn build start: # This should be production, but we need to run backend:bundle like the dockerfile does. - yarn start-backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From 732701e4a7972d424cdd9b0cb75140261c359819 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 10:42:31 +0100 Subject: [PATCH 54/74] chore: Remove the build-config flag --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index bdef387f3f..4694800e50 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -8,7 +8,7 @@ services: - yarn install build: - yarn tsc - - yarn build --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml + - yarn build update: - yarn install start: From 15932ee95b808a0e3b961ab7417aae4b22b7eadd Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 11:17:12 +0100 Subject: [PATCH 55/74] chore: try different config --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 4694800e50..e06b95d7ef 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -13,4 +13,4 @@ services: - yarn install start: # This should be production, but we need to run backend:bundle like the dockerfile does. - - yarn start-backend --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & + - node packages/backend/dist/main --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & From 1aa7817271f79fd20c168d6c362607de8a82edc1 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 11:31:27 +0100 Subject: [PATCH 56/74] chore: hopefully getting something to boot now? --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index e06b95d7ef..831ea6a9b6 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -13,4 +13,4 @@ services: - yarn install start: # This should be production, but we need to run backend:bundle like the dockerfile does. - - node packages/backend/dist/main --config app-config.yaml --config .tugboat/tugboat.app-config.production.yaml & + - yarn start-backend --config $PWD/app-config.yaml --config $PWD/.tugboat/tugboat.app-config.production.yaml & From 45f49bd12c0d1255b19850ce7cfd72df6c3042c9 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 13:48:57 +0100 Subject: [PATCH 57/74] chore: performance! --- .tugboat/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 831ea6a9b6..7ee1a775e9 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -7,8 +7,7 @@ services: init: - yarn install build: - - yarn tsc - - yarn build + - yarn workspace example-app build update: - yarn install start: From e3d850a9e460ef2c131c4bc47d5d736db95e8d72 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 13:56:37 +0100 Subject: [PATCH 58/74] chore: try skip yarn for tty problems --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 7ee1a775e9..be952d601c 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -12,4 +12,4 @@ services: - yarn install start: # This should be production, but we need to run backend:bundle like the dockerfile does. - - yarn start-backend --config $PWD/app-config.yaml --config $PWD/.tugboat/tugboat.app-config.production.yaml & + - cd packages/backend && node_modules/.bin/backstage-cli backend:dev --config ../../app-config.yaml --config ../../.tugboat/tugboat.app-config.production.yaml & From f1d1ef4635523577f640bdcc406df22ef17aed17 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 14:35:11 +0100 Subject: [PATCH 59/74] chore: updating config --- .tugboat/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index be952d601c..2678f39540 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -12,4 +12,4 @@ services: - yarn install start: # This should be production, but we need to run backend:bundle like the dockerfile does. - - cd packages/backend && node_modules/.bin/backstage-cli backend:dev --config ../../app-config.yaml --config ../../.tugboat/tugboat.app-config.production.yaml & + - cd packages/backend && node_modules/.bin/backstage-cli backend:dev --config ../../app-config.yaml --config ../../.tugboat/tugboat.app-config.production.yaml From 43290ea99126444cb09723c65e53cc6489210f76 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Tue, 26 Jan 2021 17:35:09 -0500 Subject: [PATCH 60/74] update version of git-url-parse to set ref for Azure DevOps --- packages/backend-common/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c698351615..e556a75062 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -43,7 +43,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "knex": "^0.21.6", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index cd7dab3444..e118a1d7f9 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -48,7 +48,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "globby": "^11.0.0", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", From 3179c98c85c0240dc4842ff6bde1682bd6a1eeca Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 14:49:04 +0100 Subject: [PATCH 61/74] chore: updating config again --- .tugboat/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.tugboat/config.yml b/.tugboat/config.yml index 2678f39540..2be043cd78 100644 --- a/.tugboat/config.yml +++ b/.tugboat/config.yml @@ -5,11 +5,11 @@ services: default: true commands: init: - - yarn install + - mkdir -p /etc/service/node + - echo "#!/bin/sh" > /etc/service/node/run + - echo "yarn --cwd ${TUGBOAT_ROOT} start-backend --config ${TUGBOAT_ROOT}/app-config.yaml --config ${TUGBOAT_ROOT}/.tugboat/tugboat.app-config.production.yaml" >> /etc/service/node/run + - chmod +x /etc/service/node/run build: - yarn workspace example-app build update: - yarn install - start: - # This should be production, but we need to run backend:bundle like the dockerfile does. - - cd packages/backend && node_modules/.bin/backstage-cli backend:dev --config ../../app-config.yaml --config ../../.tugboat/tugboat.app-config.production.yaml From e9b9fa2ffef7d9bfbe99061455cc1788083a2b15 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 16:38:27 +0100 Subject: [PATCH 62/74] chore: added a note about .dockerignore --- docs/getting-started/deployment-other.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 6436bdb195..92c5622593 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -62,8 +62,26 @@ COPY app-config.yaml app-config.production.yaml ./ CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"] ``` -You can add the Dockerfile to the root of your project, and run the following to -build the container under a specified tag. +Before building you should also include a `.dockerignore`. This will greatly +improve the context bootup of Docker as we are no longer sending all of the +`node_modules` into the context. It also helps us avoid some limitations and +errors that may occur when trying to share the `node_modules` folder to inside +the build. + +You can add the following contents to the root of your repository at +`.dockerignore` and it might look something like the following: + +```dockerignore +.git +node_modules +packages/*/node_modules +plugins/*/node_modules +plugins/*/dist +``` + +Once you have added both the `Dockerfile` and `.dockerignore` to the root of +your project, and run the following to build the container under a specified +tag. ```sh $ docker build -t example-deployment . From 73c299c9ee00d9475602777c157c08eae6195ca3 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Wed, 27 Jan 2021 11:54:03 -0500 Subject: [PATCH 63/74] upgrade git-url-parse to 11.4.4 --- packages/backend-common/package.json | 2 +- packages/integration/package.json | 2 +- packages/techdocs-common/package.json | 2 +- plugins/catalog-backend/package.json | 2 +- plugins/catalog-import/package.json | 2 +- plugins/catalog/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- plugins/scaffolder/package.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c698351615..e556a75062 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -43,7 +43,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", "knex": "^0.21.6", diff --git a/packages/integration/package.json b/packages/integration/package.json index fd5268dced..6c96a8562b 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -31,7 +31,7 @@ "dependencies": { "@backstage/config": "^0.1.2", "cross-fetch": "^3.0.6", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "@octokit/rest": "^18.0.12", "@octokit/auth-app": "^2.10.5", "luxon": "^1.25.0" diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 93825511cb..dd4e4d8c35 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -48,7 +48,7 @@ "dockerode": "^3.2.1", "express": "^4.17.1", "fs-extra": "^9.0.1", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "js-yaml": "^4.0.0", "json5": "^2.1.3", "mime-types": "^2.1.27", diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index b2db27770b..eaf1f69938 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -43,7 +43,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "knex": "^0.21.6", "ldapjs": "^2.2.0", "lodash": "^4.17.15", diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index b2c9991bb8..2f9784a23e 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -39,7 +39,7 @@ "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.45", "@octokit/rest": "^18.0.12", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "react": "^16.13.1", "react-dom": "^16.13.1", "react-hook-form": "^6.6.0", diff --git a/plugins/catalog/package.json b/plugins/catalog/package.json index be1681c596..0887930726 100644 --- a/plugins/catalog/package.json +++ b/plugins/catalog/package.json @@ -40,7 +40,7 @@ "@material-ui/lab": "4.0.0-alpha.45", "@types/react": "^16.9", "classnames": "^2.2.6", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "moment": "^2.26.0", "react": "^16.13.1", "react-dom": "^16.13.1", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index cd7dab3444..e118a1d7f9 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -48,7 +48,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "globby": "^11.0.0", "helmet": "^4.0.0", "isomorphic-git": "^1.8.0", diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index 3c98972aa8..483dc548ea 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -40,7 +40,7 @@ "@rjsf/core": "^2.4.0", "@rjsf/material-ui": "^2.4.0", "classnames": "^2.2.6", - "git-url-parse": "^11.4.3", + "git-url-parse": "^11.4.4", "moment": "^2.26.0", "react": "^16.13.1", "react-dom": "^16.13.1", From 9dd057662453459db97582657861ec996c28bd8c Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Wed, 27 Jan 2021 12:00:32 -0500 Subject: [PATCH 64/74] changeset --- .changeset/empty-coats-film.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .changeset/empty-coats-film.md diff --git a/.changeset/empty-coats-film.md b/.changeset/empty-coats-film.md new file mode 100644 index 0000000000..02b0225102 --- /dev/null +++ b/.changeset/empty-coats-film.md @@ -0,0 +1,12 @@ +--- +'@backstage/backend-common': patch +'@backstage/integration': patch +'@backstage/techdocs-common': patch +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-catalog-import': patch +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Upgrade [git-url-parse](https://www.npmjs.com/package/git-url-parse) to [v11.4.4](https://github.com/IonicaBizau/git-url-parse/pull/125) which fixes parsing an Azure DevOps branch ref. From e346f96540de9107f7775b5f8523e766023552f7 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Wed, 27 Jan 2021 14:17:19 -0500 Subject: [PATCH 65/74] forgot to push yarn.lock --- yarn.lock | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yarn.lock b/yarn.lock index 7c743bc5b3..1dcbe1733d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14180,6 +14180,13 @@ git-url-parse@^11.4.3: dependencies: git-up "^4.0.0" +git-url-parse@^11.4.4: + version "11.4.4" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.4.tgz#5d747debc2469c17bc385719f7d0427802d83d77" + integrity sha512-Y4o9o7vQngQDIU9IjyCmRJBin5iYjI5u9ZITnddRZpD7dcCFQj2sL2XuMNbLRE4b4B/4ENPsp2Q8P44fjAZ0Pw== + dependencies: + git-up "^4.0.0" + gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" From 8f59f5bbc21236b3b50a05273166e5e3cf559fa3 Mon Sep 17 00:00:00 2001 From: Alan Crosswell Date: Wed, 27 Jan 2021 14:19:58 -0500 Subject: [PATCH 66/74] forgot to push yarn.lock --- yarn.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn.lock b/yarn.lock index 36a133fd37..6b634511b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14180,10 +14180,10 @@ git-url-parse@^11.1.2: dependencies: git-up "^4.0.0" -git-url-parse@^11.4.3: - version "11.4.3" - resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.3.tgz#1610284edf1f14964180f5b3399ec68b692cfd87" - integrity sha512-LZTTk0nqJnKN48YRtOpR8H5SEfp1oM2tls90NuZmBxN95PnCvmuXGzqQ4QmVirBgKx2KPYfPGteX3/raWjKenQ== +git-url-parse@^11.4.4: + version "11.4.4" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.4.tgz#5d747debc2469c17bc385719f7d0427802d83d77" + integrity sha512-Y4o9o7vQngQDIU9IjyCmRJBin5iYjI5u9ZITnddRZpD7dcCFQj2sL2XuMNbLRE4b4B/4ENPsp2Q8P44fjAZ0Pw== dependencies: git-up "^4.0.0" From 1f675381a9143ec63037270f2821ea5c51dedb64 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 27 Jan 2021 21:59:44 +0100 Subject: [PATCH 67/74] chore: fixing vale stuff --- docs/getting-started/deployment-other.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/getting-started/deployment-other.md b/docs/getting-started/deployment-other.md index 92c5622593..418a6b3dd3 100644 --- a/docs/getting-started/deployment-other.md +++ b/docs/getting-started/deployment-other.md @@ -63,8 +63,8 @@ CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app ``` Before building you should also include a `.dockerignore`. This will greatly -improve the context bootup of Docker as we are no longer sending all of the -`node_modules` into the context. It also helps us avoid some limitations and +improve the context boot up time of Docker as we are no longer sending all of +the `node_modules` into the context. It also helps us avoid some limitations and errors that may occur when trying to share the `node_modules` folder to inside the build. From 83a879c4cb4debe5531074be22acc5b430fc27e5 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 00:01:55 +0100 Subject: [PATCH 68/74] chore: add a simple github action workflow --- .github/workflows/tugboat.yml | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/tugboat.yml diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml new file mode 100644 index 0000000000..4c52ce99d6 --- /dev/null +++ b/.github/workflows/tugboat.yml @@ -0,0 +1,57 @@ +name: Tugboat E2E Tests +on: deployment_status + +jobs: + run: + # When the deployment event is success + if: github.event.deployment_status.state == 'success' + name: Run test suite against tugboat + runs-on: ubuntu-latest + steps: + # Set an initial commit status message to indicate that the tests are + # running. + - name: set pending status + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + debug: true + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: 'pending', + context: 'Nightwatch.js tests', + description: 'Running tests', + target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + }); + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '14' + - name: get deployment status + id: get-status-env + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + result-encoding: string + script: | + const result = await github.repos.getDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: context.payload.deployment.id, + status_id: context.payload.deployment_status.id, + headers: { + 'Accept': 'application/vnd.github.ant-man-preview+json' + }, + }); + console.log(result); + return result.data.environment_url; + - name: echo tugboat preview url + run: | + echo ${{ steps.get-status-env.outputs.result }} + # The first time you hit a Tugboat URL it can take a while to load, so + # we visit it once here to prime it. Otherwise the very first test + # will often timeout. + curl ${{ steps.get-status-env.outputs.result }} + From 3148751354df0cb62324d65ef64c66f90e1c53dc Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 00:12:52 +0100 Subject: [PATCH 69/74] chore: update workfloew --- .github/workflows/tugboat.yml | 107 ++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml index 4c52ce99d6..da67be58b4 100644 --- a/.github/workflows/tugboat.yml +++ b/.github/workflows/tugboat.yml @@ -2,56 +2,63 @@ name: Tugboat E2E Tests on: deployment_status jobs: - run: - # When the deployment event is success + run-tests: + # Only run after a successful Tugboat deployment. if: github.event.deployment_status.state == 'success' - name: Run test suite against tugboat + name: Run tests against Tugboat runs-on: ubuntu-latest - steps: - # Set an initial commit status message to indicate that the tests are - # running. - - name: set pending status - uses: actions/github-script@v3 - with: - github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} - debug: true - script: | - return github.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: 'pending', - context: 'Nightwatch.js tests', - description: 'Running tests', - target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - }); - - uses: actions/checkout@v1 - - uses: actions/setup-node@v1 - with: - node-version: '14' - - name: get deployment status - id: get-status-env - uses: actions/github-script@v3 - with: - github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} - result-encoding: string - script: | - const result = await github.repos.getDeploymentStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - deployment_id: context.payload.deployment.id, - status_id: context.payload.deployment_status.id, - headers: { - 'Accept': 'application/vnd.github.ant-man-preview+json' - }, - }); - console.log(result); - return result.data.environment_url; - - name: echo tugboat preview url - run: | - echo ${{ steps.get-status-env.outputs.result }} - # The first time you hit a Tugboat URL it can take a while to load, so - # we visit it once here to prime it. Otherwise the very first test - # will often timeout. - curl ${{ steps.get-status-env.outputs.result }} + steps: + # Set an initial commit status message to indicate that the tests are + # running. + - name: set pending status + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + debug: true + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: 'pending', + context: 'Nightwatch.js tests', + description: 'Running tests', + target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + }); + - uses: actions/checkout@v1 + - uses: actions/setup-node@v1 + with: + node-version: '14' + + # This is required because the environment_url param that Tugboat uses + # to tell us where the preview is located isn't supported unless you + # specify the custom Accept header when getting the deployment_status, + # and GitHub actions doesn't do that by default. So instead we have to + # load the status object manually and get the data we need. + # https://developer.github.com/changes/2016-04-06-deployment-and-deployment-status-enhancements/ + - name: get deployment status + id: get-status-env + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + result-encoding: string + script: | + const result = await github.repos.getDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: context.payload.deployment.id, + status_id: context.payload.deployment_status.id, + headers: { + 'Accept': 'application/vnd.github.ant-man-preview+json' + }, + }); + console.log(result); + return result.data.environment_url; + - name: echo tugboat preview url + run: | + echo ${{ steps.get-status-env.outputs.result }} + # The first time you hit a Tugboat URL it can take a while to load, so + # we visit it once here to prime it. Otherwise the very first test + # will often timeout. + curl ${{ steps.get-status-env.outputs.result }} From fca9de6c93b53395a994bfa2271f3e5ba84d7c3f Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 00:24:28 +0100 Subject: [PATCH 70/74] chore: update workflow again --- .github/workflows/tugboat.yml | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml index da67be58b4..53fcd2e873 100644 --- a/.github/workflows/tugboat.yml +++ b/.github/workflows/tugboat.yml @@ -1,7 +1,29 @@ name: Tugboat E2E Tests on: deployment_status - jobs: + set-pending: + if: github.event.deployment_status.state != 'success' && github.event.deployment_status.state != 'failed' + name: Run tests against Tugboat + runs-on: ubuntu-latest + steps: + # Set an initial commit status message to indicate that the tests are + # running. + - name: set pending status + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + debug: true + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: 'pending', + context: 'Backstage Tugboat E2E Tests', + description: 'Running tests', + target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" + }); + run-tests: # Only run after a successful Tugboat deployment. if: github.event.deployment_status.state == 'success' @@ -21,9 +43,9 @@ jobs: repo: context.repo.repo, sha: context.sha, state: 'pending', - context: 'Nightwatch.js tests', + context: 'Backstage Tugboat E2E Tests', description: 'Running tests', - target_url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" }); - uses: actions/checkout@v1 @@ -57,8 +79,8 @@ jobs: return result.data.environment_url; - name: echo tugboat preview url run: | - echo ${{ steps.get-status-env.outputs.result }} + echo ${{steps.get-status-env.outputs.result}} # The first time you hit a Tugboat URL it can take a while to load, so # we visit it once here to prime it. Otherwise the very first test # will often timeout. - curl ${{ steps.get-status-env.outputs.result }} + curl ${{steps.get-status-env.outputs.result}} From a4716e47276475f2424471df7fd126bcc9f484bb Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 00:50:44 +0100 Subject: [PATCH 71/74] chore: fixing deployments for real --- .github/workflows/tugboat.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml index 53fcd2e873..cce4df01de 100644 --- a/.github/workflows/tugboat.yml +++ b/.github/workflows/tugboat.yml @@ -3,7 +3,7 @@ on: deployment_status jobs: set-pending: if: github.event.deployment_status.state != 'success' && github.event.deployment_status.state != 'failed' - name: Run tests against Tugboat + name: Set pending waiting for Tugboat runs-on: ubuntu-latest steps: # Set an initial commit status message to indicate that the tests are @@ -20,14 +20,14 @@ jobs: sha: context.sha, state: 'pending', context: 'Backstage Tugboat E2E Tests', - description: 'Running tests', + description: 'Waiting for Tugboat to complete deployment', target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" }); run-tests: # Only run after a successful Tugboat deployment. if: github.event.deployment_status.state == 'success' - name: Run tests against Tugboat + name: Run tests against Tugboat deployment runs-on: ubuntu-latest steps: # Set an initial commit status message to indicate that the tests are @@ -44,7 +44,7 @@ jobs: sha: context.sha, state: 'pending', context: 'Backstage Tugboat E2E Tests', - description: 'Running tests', + description: 'Running against tugboat preview', target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" }); @@ -79,8 +79,4 @@ jobs: return result.data.environment_url; - name: echo tugboat preview url run: | - echo ${{steps.get-status-env.outputs.result}} - # The first time you hit a Tugboat URL it can take a while to load, so - # we visit it once here to prime it. Otherwise the very first test - # will often timeout. curl ${{steps.get-status-env.outputs.result}} From 408962dcead8301ae3885a7769264f117ea83fe0 Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 01:17:52 +0100 Subject: [PATCH 72/74] chore: last github workflow plz --- .github/workflows/tugboat.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml index cce4df01de..3d6980c81e 100644 --- a/.github/workflows/tugboat.yml +++ b/.github/workflows/tugboat.yml @@ -80,3 +80,32 @@ jobs: - name: echo tugboat preview url run: | curl ${{steps.get-status-env.outputs.result}} + # Update the commit status with a fail or success. + - name: set status + if: ${{ success() }} + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: "success", + context: 'Backstage Tugboat E2E Tests', + target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" + }); + - name: set status + if: ${{ failure() }} || ${{ cancelled() }} + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: "error", + context: 'Backstage Tugboat E2E Tests', + target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" + }); From f2b7817d4f9cca4740da984b819a905573349531 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Jan 2021 04:48:11 +0000 Subject: [PATCH 73/74] chore(deps): bump core-js from 3.6.5 to 3.8.3 Bumps [core-js](https://github.com/zloirock/core-js) from 3.6.5 to 3.8.3. - [Release notes](https://github.com/zloirock/core-js/releases) - [Changelog](https://github.com/zloirock/core-js/blob/master/CHANGELOG.md) - [Commits](https://github.com/zloirock/core-js/compare/v3.6.5...v3.8.3) Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6b634511b4..aefde4902f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10787,9 +10787,9 @@ core-js-pure@^3.0.0, core-js-pure@^3.0.1: integrity sha512-epIhRLkXdgv32xIUFaaAry2wdxZYBi6bgM7cB136dzzXXa+dFyRLTZeLUJxnd8ShrmyVXBub63n2NHo2JAt8Cw== core-js@3, core-js@^3.0.1, core-js@^3.0.4, core-js@^3.5.0, core-js@^3.6.0, core-js@^3.6.5: - version "3.6.5" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.6.5.tgz#7395dc273af37fb2e50e9bd3d9fe841285231d1a" - integrity sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA== + version "3.8.3" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.8.3.tgz#c21906e1f14f3689f93abcc6e26883550dd92dd0" + integrity sha512-KPYXeVZYemC2TkNEkX/01I+7yd+nX3KddKwZ1Ww7SKWdI2wQprSgLmrTddT8nw92AjEklTsPBoSdQBhbI1bQ6Q== core-js@^2.4.0, core-js@^2.5.7, core-js@^2.6.10, core-js@^2.6.5: version "2.6.11" From 99460a64371c4b2604e151088f86463fb6635c2b Mon Sep 17 00:00:00 2001 From: blam Date: Thu, 28 Jan 2021 10:45:16 +0100 Subject: [PATCH 74/74] chore: updating workflow --- .github/workflows/tugboat.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/tugboat.yml b/.github/workflows/tugboat.yml index 3d6980c81e..fdb2e1ddc4 100644 --- a/.github/workflows/tugboat.yml +++ b/.github/workflows/tugboat.yml @@ -80,7 +80,20 @@ jobs: - name: echo tugboat preview url run: | curl ${{steps.get-status-env.outputs.result}} - # Update the commit status with a fail or success. + - name: set status + if: ${{ failure() }} + uses: actions/github-script@v3 + with: + github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} + script: | + return github.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: "error", + context: 'Backstage Tugboat E2E Tests', + target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" + }); - name: set status if: ${{ success() }} uses: actions/github-script@v3 @@ -95,17 +108,3 @@ jobs: context: 'Backstage Tugboat E2E Tests', target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" }); - - name: set status - if: ${{ failure() }} || ${{ cancelled() }} - uses: actions/github-script@v3 - with: - github-token: ${{secrets.GH_SERVICE_ACCOUNT_TOKEN}} - script: | - return github.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: "error", - context: 'Backstage Tugboat E2E Tests', - target_url: "https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" - });