paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 36 Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:backstage/backstage into mob/scaffolder-frontend

Browse Source 
* 'master' of github.com:backstage/backstage: (118 commits)
  cli: Fix handling of dynamic imports in esm.js files
  minor typo in migration
  chore(deps): bump archiver from 5.1.0 to 5.2.0
  dockerfile: mention build-image command
  Apply suggestions from code review
  update backend Dockerfile to use config example and fix comment
  docs: add full docker deployment docs
  chore: fix code review
  chore: fixing syntax
  docs: fixing custom implementations of utitiy apis
  a small start to the integrations section of the config
  TechDocs: Add changeset about Docker permission fix
  Updated unit tests for the new UI
  TechDocs: Pass user and group ID when invoking docker container
  Replace logging erro and return undefined for a throw new Error
  @types/react 16 not 17
  Use a more strict type for `variant` of cards
  docs(TechDocs): Add more context with AWS docs hyperlinks
  Added missing dep on @types/react
  Removed unused import
  ...
This commit is contained in:
blam
2021-02-18 11:41:34 +01:00
parent 2cf720547c 5dc7fa2773
commit d50ddbd1d3
156 changed files with 6442 additions and 1344 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app/src/components/catalog/EntityPage.tsx
+62 -2
View File
@@ -16,8 +16,10 @@
import {
ApiEntity,
DomainEntity,
Entity,
GroupEntity,
SystemEntity,
UserEntity,
} from '@backstage/catalog-model';
import { EmptyState } from '@backstage/core';
@@ -25,15 +27,19 @@ import {
ApiDefinitionCard,
ConsumedApisCard,
ConsumingComponentsCard,
EntityHasApisCard,
ProvidedApisCard,
ProvidingComponentsCard,
} from '@backstage/plugin-api-docs';
import {
AboutCard,
EntityHasComponentsCard,
EntityHasSubcomponentsCard,
EntityHasSystemsCard,
EntityLinksCard,
EntityPageLayout,
} from '@backstage/plugin-catalog';
import { useEntity } from '@backstage/plugin-catalog-react';
import { EntityProvider, useEntity } from '@backstage/plugin-catalog-react';
import {
isPluginApplicableToEntity as isCircleCIAvailable,
Router as CircleCIRouter,
@@ -178,7 +184,9 @@ const ComponentOverviewContent = ({ entity }: { entity: Entity }) => (
</Grid>
{isPagerDutyAvailable(entity) && (
<Grid item md={6}>
<PagerDutyCard entity={entity} />
<EntityProvider entity={entity}>
<PagerDutyCard />
</EntityProvider>
</Grid>
)}
<Grid item md={4} sm={6}>
@@ -206,6 +214,9 @@ const ComponentOverviewContent = ({ entity }: { entity: Entity }) => (
<PullRequestsStatsCard entity={entity} />
</Grid>
)}
<Grid item md={6}>
<EntityHasSubcomponentsCard variant="gridItem" />
</Grid>
</Grid>
);
@@ -425,6 +436,51 @@ const GroupEntityPage = ({ entity }: { entity: Entity }) => (
</EntityPageLayout>
);
const SystemOverviewContent = ({ entity }: { entity: SystemEntity }) => (
<Grid container spacing={3} alignItems="stretch">
<Grid item md={6}>
<AboutCard entity={entity} variant="gridItem" />
</Grid>
<Grid item md={6}>
<EntityHasComponentsCard variant="gridItem" />
</Grid>
<Grid item md={6}>
<EntityHasApisCard variant="gridItem" />
</Grid>
</Grid>
);
const SystemEntityPage = ({ entity }: { entity: Entity }) => (
<EntityPageLayout>
<EntityPageLayout.Content
path="/*"
title="Overview"
element={<SystemOverviewContent entity={entity as SystemEntity} />}
/>
</EntityPageLayout>
);
const DomainOverviewContent = ({ entity }: { entity: DomainEntity }) => (
<Grid container spacing={3} alignItems="stretch">
<Grid item md={6}>
<AboutCard entity={entity} variant="gridItem" />
</Grid>
<Grid item md={6}>
<EntityHasSystemsCard variant="gridItem" />
</Grid>
</Grid>
);
const DomainEntityPage = ({ entity }: { entity: Entity }) => (
<EntityPageLayout>
<EntityPageLayout.Content
path="/*"
title="Overview"
element={<DomainOverviewContent entity={entity as DomainEntity} />}
/>
</EntityPageLayout>
);
export const EntityPage = () => {
const { entity } = useEntity();
@@ -437,6 +493,10 @@ export const EntityPage = () => {
return <GroupEntityPage entity={entity} />;
case 'user':
return <UserEntityPage entity={entity} />;
case 'system':
return <SystemEntityPage entity={entity} />;
case 'domain':
return <DomainEntityPage entity={entity} />;
default:
return <DefaultEntityPage entity={entity} />;
}
packages/backend/Dockerfile
+17 -7
View File
@@ -1,16 +1,26 @@
FROM node:14-buster
# This dockerfile builds an image for the backend package.
# It should be executed with the root of the repo as docker context.
#
# Before building this image, be sure to have run the following commands in the repo root:
#
# yarn install
# yarn tsc
# yarn build
#
# Once the commands have been run, you can build the image using `yarn build-image`
WORKDIR /usr/src/app
FROM node:14-buster-slim
WORKDIR /app
# Copy repo skeleton first, to avoid unnecessary docker cache invalidation.
# The skeleton contains the package.json of each package in the monorepo,
# and along with yarn.lock and the root package.json, that's enough to run yarn install.
ADD yarn.lock package.json skeleton.tar ./
ADD yarn.lock package.json packages/backend/dist/skeleton.tar.gz ./
RUN yarn install --frozen-lockfile --production --network-timeout 300000 && rm -rf "$(yarn cache dir)"
# This will copy the contents of the dist-workspace when running the build-image command.
# Do not use this Dockerfile outside of that command, as it will copy in the source code instead.
COPY . .
# Then copy the rest of the backend bundle, along with any other files we might want.
ADD packages/backend/dist/bundle.tar.gz app-config.yaml ./
CMD ["node", "packages/backend"]
CMD ["node", "packages/backend", "--config", "app-config.yaml"]
packages/backend/package.json
+2 -2
View File
@@ -18,8 +18,8 @@
"backstage"
],
"scripts": {
"build": "backstage-cli backend:build",
"build-image": "backstage-cli backend:build-image --build --tag example-backend",
"build": "backstage-cli backend:bundle",
"build-image": "docker build ../.. -f Dockerfile --tag example-backend",
"start": "backstage-cli backend:dev",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
packages/cli/config/jest.js
+1 -3
View File
@@ -83,10 +83,8 @@ async function getConfig() {
},
},
// We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed
// TODO: jest is working on module support, it's possible that we can remove this in the future
transform: {
'\\.esm\\.js$': require.resolve('jest-esm-transformer'),
'\\.esm\\.js$': require.resolve('./jestEsmTransform.js'), // See jestEsmTransform.js
'\\.(js|jsx|ts|tsx)$': require.resolve('ts-jest'),
'\\.(bmp|gif|jpg|jpeg|png|frag|xml|svg)$': require.resolve(
'./jestFileTransform.js',
packages/cli/config/jestEsmTransform.js
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const babel = require('@babel/core');
// We build .esm.js files with plugin:build, so to be able to load these in tests they need to be transformed
// TODO: jest is working on module support, it's possible that we can remove this in the future
module.exports = {
process(src) {
const result = babel.transform(src, {
babelrc: false,
compact: false,
plugins: [
// This transforms the regular ESM syntax, import and export statements
require.resolve('@babel/plugin-transform-modules-commonjs'),
// This transforms dynamic `import()`, which is not supported yet in the Node.js VM API
require.resolve('babel-plugin-dynamic-import-node'),
],
});
return result.code;
},
};
packages/cli/package.json
+3 -1
View File
@@ -28,6 +28,8 @@
"backstage-cli": "bin/backstage-cli"
},
"dependencies": {
"@babel/core": "^7.4.4",
"@babel/plugin-transform-modules-commonjs": "^7.4.4",
"@backstage/cli-common": "^0.1.1",
"@backstage/config": "^0.1.2",
"@backstage/config-loader": "^0.5.1",
@@ -53,6 +55,7 @@
"@typescript-eslint/eslint-plugin": "^v4.14.0",
"@typescript-eslint/parser": "^v4.14.0",
"@yarnpkg/lockfile": "^1.1.0",
"babel-plugin-dynamic-import-node": "^2.3.3",
"bfj": "^7.0.2",
"chalk": "^4.0.0",
"chokidar": "^3.3.1",
@@ -78,7 +81,6 @@
"inquirer": "^7.0.4",
"jest": "^26.0.1",
"jest-css-modules": "^2.1.0",
"jest-esm-transformer": "^1.0.0",
"lodash": "^4.17.19",
"mini-css-extract-plugin": "^0.9.0",
"ora": "^4.0.3",
packages/core/src/components/ProgressBars/GaugeCard.tsx
+2 -2
View File
@@ -16,14 +16,14 @@
import React from 'react';
import { makeStyles } from '@material-ui/core';
import { InfoCard } from '../../layout/InfoCard';
import { InfoCard, InfoCardVariants } from '../../layout/InfoCard';
import { BottomLinkProps } from '../../layout/BottomLink';
import { Gauge } from './Gauge';
type Props = {
title: string;
subheader?: string;
variant?: string;
variant?: InfoCardVariants;
/** Progress in % specified as decimal, e.g. "0.23" */
progress: number;
deepLink?: BottomLinkProps;
packages/core/src/components/Table/Table.stories.tsx
+68 -9
View File
@@ -14,8 +14,10 @@
* limitations under the License.
*/
import { makeStyles } from '@material-ui/core';
import React from 'react';
import { Table, SubvalueCell, TableColumn } from './';
import { Link } from '../Link';
import { SubvalueCell, Table, TableColumn } from './';
import { TableFilter } from './Table';
export default {
@@ -23,7 +25,16 @@ export default {
component: Table,
};
const containerStyle = { width: 850 };
const useStyles = makeStyles(theme => ({
container: {
width: 850,
},
empty: {
padding: theme.spacing(2),
display: 'flex',
justifyContent: 'center',
},
}));
const generateTestData: (number: number) => Array<{}> = (rows = 10) => {
const data: Array<{}> = [];
@@ -43,6 +54,7 @@ const generateTestData: (number: number) => Array<{}> = (rows = 10) => {
const testData10 = generateTestData(10);
export const DefaultTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -66,7 +78,7 @@ export const DefaultTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table
options={{ paging: false }}
data={testData10}
@@ -77,7 +89,8 @@ export const DefaultTable = () => {
);
};
export const SubtitleTable = () => {
export const EmptyTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -101,7 +114,49 @@ export const SubtitleTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table
options={{ paging: false }}
data={[]}
columns={columns}
emptyContent={
<div className={classes.empty}>
No data was added yet,&nbsp;
<Link to="http://backstage.io/">learn how to add data</Link>.
</div>
}
title="Backstage Table"
/>
</div>
);
};
export const SubtitleTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
field: 'col1',
highlight: true,
},
{
title: 'Column 2',
field: 'col2',
},
{
title: 'Numeric value',
field: 'number',
type: 'numeric',
},
{
title: 'A Date',
field: 'date',
type: 'date',
},
];
return (
<div className={classes.container}>
<Table
options={{ paging: false }}
data={testData10}
@@ -114,6 +169,7 @@ export const SubtitleTable = () => {
};
export const HiddenSearchTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -137,7 +193,7 @@ export const HiddenSearchTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table
options={{ paging: false, search: false }}
data={testData10}
@@ -148,6 +204,7 @@ export const HiddenSearchTable = () => {
};
export const SubvalueTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -181,13 +238,14 @@ export const SubvalueTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table options={{ paging: false }} data={testData10} columns={columns} />
</div>
);
};
export const DenseTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -211,7 +269,7 @@ export const DenseTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table
options={{ paging: false, padding: 'dense' }}
data={testData10}
@@ -223,6 +281,7 @@ export const DenseTable = () => {
};
export const FilterTable = () => {
const classes = useStyles();
const columns: TableColumn[] = [
{
title: 'Column 1',
@@ -261,7 +320,7 @@ export const FilterTable = () => {
];
return (
<div style={containerStyle}>
<div className={classes.container}>
<Table
options={{ paging: false, padding: 'dense' }}
data={testData10}
packages/core/src/components/Table/Table.test.tsx
+12
View File
@@ -53,4 +53,16 @@ describe('<Table />', () => {
);
expect(rendered.getByText('subtitle')).toBeInTheDocument();
});
it('renders custom empty component if empty', async () => {
const rendered = await renderInTestApp(
<Table
subtitle="subtitle"
emptyContent={<div>EMPTY</div>}
columns={minProps.columns}
data={[]}
/>,
);
expect(rendered.getByText('EMPTY')).toBeInTheDocument();
});
});
packages/core/src/components/Table/Table.tsx
+22
View File
@@ -42,12 +42,14 @@ import MTable, {
Column,
Icons,
MaterialTableProps,
MTableBody,
MTableHeader,
MTableToolbar,
Options,
} from 'material-table';
import React, {
forwardRef,
ReactNode,
useCallback,
useEffect,
useRef,
@@ -202,6 +204,7 @@ export interface TableProps<T extends object = {}>
subtitle?: string;
filters?: TableFilter[];
initialState?: TableState;
emptyContent?: ReactNode;
onStateChange?: (state: TableState) => any;
}
@@ -212,6 +215,7 @@ export function Table<T extends object = {}>({
subtitle,
filters,
initialState,
emptyContent,
onStateChange,
...props
}: TableProps<T>) {
@@ -423,6 +427,23 @@ export function Table<T extends object = {}>({
],
);
const Body = useCallback(
bodyProps => {
if (emptyContent && data.length === 0) {
return (
<tbody>
<tr>
<td colSpan={columns.length}>{emptyContent}</td>
</tr>
</tbody>
);
}
return <MTableBody {...bodyProps} />;
},
[data, emptyContent, columns],
);
return (
<div className={tableClasses.root}>
{filtersOpen && data && filters?.length && (
@@ -438,6 +459,7 @@ export function Table<T extends object = {}>({
<MTableHeader classes={headerClasses} {...headerProps} />
),
Toolbar,
Body,
}}
options={{ ...defaultOptions, ...options }}
columns={MTColumns}
packages/core/src/layout/InfoCard/InfoCard.tsx
+3 -1
View File
@@ -87,6 +87,8 @@ const VARIANT_STYLES = {
},
};
export type InfoCardVariants = 'flex' | 'fullHeight' | 'gridItem';
/**
* InfoCard is used to display a paper-styled block on the screen, similar to a panel.
*
@@ -111,7 +113,7 @@ type Props = {
divider?: boolean;
deepLink?: BottomLinkProps;
slackChannel?: string;
variant?: string;
variant?: InfoCardVariants;
style?: object;
cardStyle?: object;
children?: ReactNode;
packages/core/src/layout/InfoCard/index.ts
+1
View File
@@ -15,3 +15,4 @@
*/
export { InfoCard } from './InfoCard';
export type { InfoCardVariants } from './InfoCard';
packages/create-app/templates/default-app/.dockerignore
+5
View File
@@ -0,0 +1,5 @@
.git
node_modules
packages
!packages/backend/dist
plugins
packages/create-app/templates/default-app/packages/backend/Dockerfile
+17 -7
View File
@@ -1,16 +1,26 @@
FROM node:12-buster
# This dockerfile builds an image for the backend package.
# It should be executed with the root of the repo as docker context.
#
# Before building this image, be sure to have run the following commands in the repo root:
#
# yarn install
# yarn tsc
# yarn build
#
# Once the commands have been run, you can build the image using `yarn build-image`
WORKDIR /usr/src/app
FROM node:14-buster-slim
WORKDIR /app
# Copy repo skeleton first, to avoid unnecessary docker cache invalidation.
# The skeleton contains the package.json of each package in the monorepo,
# and along with yarn.lock and the root package.json, that's enough to run yarn install.
ADD yarn.lock package.json skeleton.tar ./
ADD yarn.lock package.json packages/backend/dist/skeleton.tar.gz ./
RUN yarn install --frozen-lockfile --production --network-timeout 300000 && rm -rf "$(yarn cache dir)"
# This will copy the contents of the dist-workspace when running the build-image command.
# Do not use this Dockerfile outside of that command, as it will copy in the source code instead.
COPY . .
# Then copy the rest of the backend bundle, along with any other files we might want.
ADD packages/backend/dist/bundle.tar.gz app-config.yaml ./
CMD ["node", "packages/backend", "--config", "app-config.yaml", "--config", "app-config.production.yaml"]
CMD ["node", "packages/backend", "--config", "app-config.yaml"]
packages/create-app/templates/default-app/packages/backend/package.json.hbs
+2 -2
View File
@@ -8,8 +8,8 @@
"node": "12 || 14"
},
"scripts": {
"build": "backstage-cli backend:build",
"build-image": "backstage-cli backend:build-image --build --tag backstage",
"build": "backstage-cli backend:bundle",
"build-image": "docker build ../.. -f Dockerfile --tag backstage",
"start": "backstage-cli backend:dev",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
packages/techdocs-common/__mocks__/@azure/storage-blob.ts
+64 -16
View File
@@ -14,6 +14,10 @@
* limitations under the License.
*/
import fs from 'fs';
import type {
BlobUploadCommonResponse,
ContainerGetPropertiesResponse,
} from '@azure/storage-blob';
export class BlockBlobClient {
private readonly blobName;
@@ -22,23 +26,33 @@ export class BlockBlobClient {
this.blobName = blobName;
}
uploadFile(source: string) {
return new Promise((resolve, reject) => {
if (!fs.existsSync(source)) {
reject('');
} else {
resolve('');
}
uploadFile(source: string): Promise<BlobUploadCommonResponse> {
return Promise.resolve({
_response: {
request: {
url: `https://example.blob.core.windows.net`,
} as any,
status: 200,
headers: {} as any,
},
});
}
exists() {
return new Promise((resolve, reject) => {
if (fs.existsSync(this.blobName)) {
resolve(true);
} else {
reject({ message: 'The object doest not exist !' });
}
return Promise.resolve(fs.existsSync(this.blobName));
}
}
class BlockBlobClientFailUpload extends BlockBlobClient {
uploadFile(source: string): Promise<BlobUploadCommonResponse> {
return Promise.resolve({
_response: {
request: {
url: `https://example.blob.core.windows.net`,
} as any,
status: 500,
headers: {} as any,
},
});
}
}
@@ -50,9 +64,16 @@ export class ContainerClient {
this.containerName = containerName;
}
getProperties() {
return new Promise(resolve => {
resolve('');
getProperties(): Promise<ContainerGetPropertiesResponse> {
return Promise.resolve({
_response: {
request: {
url: `https://example.blob.core.windows.net`,
} as any,
status: 200,
headers: {} as any,
parsedHeaders: {},
},
});
}
@@ -61,6 +82,27 @@ export class ContainerClient {
}
}
class ContainerClientFailGetProperties extends ContainerClient {
getProperties(): Promise<ContainerGetPropertiesResponse> {
return Promise.resolve({
_response: {
request: {
url: `https://example.blob.core.windows.net`,
} as any,
status: 404,
headers: {} as any,
parsedHeaders: {},
},
});
}
}
class ContainerClientFailUpload extends ContainerClient {
getBlockBlobClient(blobName: string) {
return new BlockBlobClientFailUpload(blobName);
}
}
export class BlobServiceClient {
private readonly url;
private readonly credential;
@@ -71,6 +113,12 @@ export class BlobServiceClient {
}
getContainerClient(containerName: string) {
if (containerName === 'bad_container') {
return new ContainerClientFailGetProperties(containerName);
}
if (this.credential.accountName === 'failupload') {
return new ContainerClientFailUpload(containerName);
}
return new ContainerClient(containerName);
}
}
packages/techdocs-common/src/stages/generate/helpers.test.ts
+27 -2
View File
@@ -30,6 +30,7 @@ import {
patchMkdocsYmlPreBuild,
runDockerContainer,
storeEtagMetadata,
UserOptions,
} from './helpers';
const mockEntity = {
@@ -114,7 +115,7 @@ describe('helpers', () => {
imageName,
args,
expect.any(Stream),
{
expect.objectContaining({
Volumes: {
'/content': {},
'/result': {},
@@ -123,7 +124,7 @@ describe('helpers', () => {
HostConfig: {
Binds: [`${docsDir}:/content`, `${outputDir}:/result`],
},
},
}),
);
});
@@ -139,6 +140,30 @@ describe('helpers', () => {
expect(mockDocker.ping).toHaveBeenCalled();
});
it('should pass through the user and group id from the host machine and set the home dir', async () => {
await runDockerContainer({
imageName,
args,
docsDir,
outputDir,
dockerClient: mockDocker,
});
const userOptions: UserOptions = {};
if (process.getuid && process.getgid) {
userOptions.User = `${process.getuid()}:${process.getgid()}`;
}
expect(mockDocker.run).toHaveBeenCalledWith(
imageName,
args,
expect.any(Stream),
expect.objectContaining({
...userOptions,
}),
);
});
describe('where docker is unavailable', () => {
const dockerError = 'a docker error';
packages/techdocs-common/src/stages/generate/helpers.ts
+18
View File
@@ -51,6 +51,12 @@ export type RunCommandOptions = {
logStream?: Writable;
};
export type UserOptions = {
User?: string;
};
// To be replaced by a runDockerContainer from backend-common
// shared between Scaffolder and TechDocs and any other plugin.
export async function runDockerContainer({
imageName,
args,
@@ -78,6 +84,17 @@ export async function runDockerContainer({
});
});
const userOptions: UserOptions = {};
// @ts-ignore
if (process.getuid && process.getgid) {
// Files that are created inside the Docker container will be owned by
// root on the host system on non Mac systems, because of reasons. Mainly the fact that
// volume sharing is done using NFS on Mac and actual mounts in Linux world.
// So we set the user in the container as the same user and group id as the host.
// On Windows we don't have process.getuid nor process.getgid
userOptions.User = `${process.getuid()}:${process.getgid()}`;
}
const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run(
imageName,
args,
@@ -91,6 +108,7 @@ export async function runDockerContainer({
HostConfig: {
Binds: [`${docsDir}:/content`, `${outputDir}:/result`],
},
...userOptions,
...createOptions,
},
);
packages/techdocs-common/src/stages/publish/awsS3.ts
+8 -7
View File
@@ -52,10 +52,13 @@ export class AwsS3Publish implements PublisherBase {
);
}
// Credentials is an optional config. If missing, default AWS environment variables
// or AWS shared credentials file at ~/.aws/credentials will be used to authenticate
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-environment.html
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/loading-node-credentials-shared.html
// Credentials is an optional config. If missing, the default ways of authenticating AWS SDK V2 will be used.
// 1. AWS environment variables
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-environment.html
// 2. AWS shared credentials file at ~/.aws/credentials
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-shared.html
// 3. IAM Roles for EC2
// https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/loading-node-credentials-iam.html
const credentials = config.getOptionalConfig(
'techdocs.publisher.awsS3.credentials',
);
@@ -67,9 +70,7 @@ export class AwsS3Publish implements PublisherBase {
}
// AWS Region is an optional config. If missing, default AWS env variable AWS_REGION
// or AWS shared credentials file at ~/.aws/credentials will be used. Any way, AWS SDK v3 client needs
// to have the AWS Region information for it to work.
// https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/setting-region.html
// or AWS shared credentials file at ~/.aws/credentials will be used.
const region = config.getOptionalString('techdocs.publisher.awsS3.region');
const storageClient = new aws.S3({
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+117 -20
View File
@@ -19,6 +19,7 @@ import { getVoidLogger } from '@backstage/backend-common';
import { AzureBlobStoragePublish } from './azureBlobStorage';
import { PublisherBase } from './types';
import type { Entity } from '@backstage/catalog-model';
import type { Logger } from 'winston';
const createMockEntity = (annotations = {}) => {
return {
@@ -43,33 +44,40 @@ const getEntityRootDir = (entity: Entity) => {
return entityRootDir;
};
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
function createLogger() {
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
return logger;
}
let publisher: PublisherBase;
beforeEach(async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
describe('publishing with valid credentials', () => {
let logger: Logger;
beforeEach(async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
containerName: 'containerName',
},
},
},
});
logger = createLogger();
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
});
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
});
describe('AzureBlobStoragePublish', () => {
describe('publish', () => {
it('should publish a directory', async () => {
const entity = createMockEntity();
@@ -116,7 +124,7 @@ describe('AzureBlobStoragePublish', () => {
})
.catch(error =>
expect(error.message).toContain(
'Unable to upload file(s) to Azure Blob Storage. Error Failed to read template directory: ENOENT, no such file or directory',
'Unable to upload file(s) to Azure Blob Storage. Failed to read template directory: ENOENT, no such file or directory',
),
);
mockFs.restore();
@@ -145,3 +153,92 @@ describe('AzureBlobStoragePublish', () => {
});
});
});
describe('error reporting', () => {
it('reports an error when unable to read container properties', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'accountName',
},
containerName: 'bad_container',
},
},
},
});
const logger = createLogger();
let error;
try {
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
} catch (e) {
error = e;
}
expect(error).toBeInstanceOf(Error);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Could not retrieve metadata about the Azure Blob Storage container bad_container.`,
),
);
});
it('reports an error when bad account credentials', async () => {
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
type: 'azureBlobStorage',
azureBlobStorage: {
credentials: {
accountName: 'failupload',
accountKey: 'accountKey',
},
containerName: 'containerName',
},
},
},
});
const logger = createLogger();
publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger);
const entity = createMockEntity();
const entityRootDir = getEntityRootDir(entity);
mockFs({
[entityRootDir]: {
'index.html': '',
},
});
let error;
try {
await publisher.publish({
entity,
directory: entityRootDir,
});
} catch (e) {
error = e;
}
expect(error.message).toContain(
`Unable to upload file(s) to Azure Blob Storage.`,
);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Upload failed for test-namespace/TestKind/test-component-name/index.html with status code 500`,
),
);
mockFs.restore();
});
});
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+54 -45
View File
@@ -17,7 +17,6 @@ import platformPath from 'path';
import express from 'express';
import {
BlobServiceClient,
BlobUploadCommonResponse,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import { DefaultAzureCredential } from '@azure/identity';
@@ -79,25 +78,25 @@ export class AzureBlobStoragePublish implements PublisherBase {
credential,
);
await storageClient
.getContainerClient(containerName)
.getProperties()
.then(() => {
logger.info(
`Successfully connected to the Azure Blob Storage container ${containerName}.`,
);
})
.catch(reason => {
logger.error(
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
try {
const response = await storageClient
.getContainerClient(containerName)
.getProperties();
if (response._response.status >= 400) {
throw new Error(
`from Azure Blob Storage client library: ${reason.message}`,
`Failed to retrieve metadata from ${response._response.request.url} with status code ${response._response.status}.`,
);
});
}
} catch (e) {
logger.error(
`Could not retrieve metadata about the Azure Blob Storage container ${containerName}. ` +
'Make sure that the Azure project and container exist and the access key is setup correctly ' +
'techdocs.publisher.azureBlobStorage.credentials defined in app config has correct permissions. ' +
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
);
throw new Error(`from Azure Blob Storage client library: ${e.message}`);
}
return new AzureBlobStoragePublish(storageClient, containerName, logger);
}
@@ -122,8 +121,6 @@ export class AzureBlobStoragePublish implements PublisherBase {
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const uploadPromises: Array<Promise<BlobUploadCommonResponse>> = [];
// Bound the number of concurrent batches. We want a bit of concurrency for
// performance reasons, but not so much that we starve the connection pool
// or start thrashing.
@@ -140,23 +137,43 @@ export class AzureBlobStoragePublish implements PublisherBase {
); // Azure Blob Storage Container file relative path
return limiter(async () => {
await uploadPromises.push(
this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath),
);
const response = await this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(filePath);
if (response._response.status >= 400) {
return {
...response,
error: new Error(
`Upload failed for ${filePath} with status code ${response._response.status}`,
),
};
}
return {
...response,
error: undefined,
};
});
});
await Promise.all(promises).then(() => {
const responses = await Promise.all(promises);
const failed = responses.filter(r => r.error);
if (failed.length === 0) {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
`Successfully uploaded the ${responses.length} generated file(s) for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
});
return;
} else {
throw new Error(
failed
.map(r => r.error?.message)
.filter(Boolean)
.join(' '),
);
}
} catch (e) {
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. Error ${e.message}`;
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e.message}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
@@ -241,19 +258,11 @@ export class AzureBlobStoragePublish implements PublisherBase {
* A helper function which checks if index.html of an Entity's docs site is available. This
* can be used to verify if there are any pre-generated docs available to serve.
*/
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
return new Promise(resolve => {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(`${entityRootDir}/index.html`)
.exists()
.then((response: boolean) => {
resolve(response);
})
.catch(() => {
resolve(false);
});
});
hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
return this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(`${entityRootDir}/index.html`)
.exists();
}
}