- (
- {children}
- ),
- },
- ]}
- />,
- ),
+ const rendered = await renderInTestApp(
+ (
+ {children}
+ ),
+ },
+ ]}
+ />,
);
expect(onClickFunction).not.toHaveBeenCalled();
diff --git a/packages/core/src/layout/HeaderLabel/HeaderLabel.test.tsx b/packages/core/src/layout/HeaderLabel/HeaderLabel.test.tsx
index fdc6ef8c6e..acadb22525 100644
--- a/packages/core/src/layout/HeaderLabel/HeaderLabel.test.tsx
+++ b/packages/core/src/layout/HeaderLabel/HeaderLabel.test.tsx
@@ -15,38 +15,37 @@
*/
import React from 'react';
-import { render } from '@testing-library/react';
-import { wrapInTestApp } from '@backstage/test-utils';
+import { renderInTestApp } from '@backstage/test-utils';
import { HeaderLabel } from './HeaderLabel';
describe('')).toBeInTheDocument();
});
- it('should say unknown when passing null as value prop', () => {
- const rendered = render(
- wrapInTestApp(')).toBeInTheDocument();
});
- it('should have value', () => {
- const rendered = render(
- wrapInTestApp(...
- *
- * Variants can be combined in a whitespace delimited list like so:
- *
- * ...
+ * ...
*/
type Props = {
title?: ReactNode;
@@ -163,17 +158,21 @@ export const InfoCard = ({
noPadding,
}: Props): JSX.Element => {
const classes = useStyles();
-
/**
* If variant is specified, we build up styles for that particular variant for both
* the Card and the CardContent (since these need to be synced)
*/
let calculatedStyle = {};
let calculatedCardStyle = {};
-
if (variant) {
const variants = variant.split(/[\s]+/g);
variants.forEach(name => {
+ if (name === 'height100') {
+ // eslint-disable-next-line no-console
+ console.warn(
+ "Variant 'height100' of InfoCard is deprecated. Use variant 'gridItem' instead.",
+ );
+ }
calculatedStyle = {
...calculatedStyle,
...VARIANT_STYLES.card[name as keyof typeof VARIANT_STYLES['card']],
diff --git a/packages/core/src/layout/ItemCard/ItemCard.stories.tsx b/packages/core/src/layout/ItemCard/ItemCard.stories.tsx
index 46b93da370..75031a0765 100644
--- a/packages/core/src/layout/ItemCard/ItemCard.stories.tsx
+++ b/packages/core/src/layout/ItemCard/ItemCard.stories.tsx
@@ -18,13 +18,13 @@ import { ItemCard } from '.';
import { Grid } from '@material-ui/core';
export default {
- title: 'Item Card',
+ title: 'Layout/Item Card',
component: ItemCard,
};
export const Default = () => (
-
+
(
onClick={() => {}}
/>
-
+
(
export const Tags = () => (
-
+
-
+
= ({
{title}
- {tags?.map(tag => (
-
+ {tags?.map((tag, i) => (
+
))}
{description}
diff --git a/packages/core/src/layout/Page/Page.stories.tsx b/packages/core/src/layout/Page/Page.stories.tsx
index 8fd779196d..3b1dd07c7a 100644
--- a/packages/core/src/layout/Page/Page.stories.tsx
+++ b/packages/core/src/layout/Page/Page.stories.tsx
@@ -21,7 +21,6 @@ import {
HeaderLabel,
ContentHeader,
Content,
- pageTheme,
InfoCard,
HeaderTabs,
} from '../';
@@ -36,7 +35,7 @@ import {
import { Box, Typography, Link, Chip, Grid } from '@material-ui/core';
export default {
- title: 'Example Plugin',
+ title: 'Plugins/Examples',
component: Page,
};
@@ -119,14 +118,14 @@ const DataGrid = () => (
justify="space-between"
direction="row"
>
-
+
-
+
(
);
const ExampleHeader = () => (
-
+
@@ -196,7 +195,7 @@ export const PluginWithData = () => {
const [selectedTab, setSelectedTab] = useState(2);
return (
-
+
{
export const PluginWithTable = () => {
return (
-
+
(pageTheme.home);
+import { BackstageTheme } from '@backstage/theme';
+import { makeStyles, ThemeProvider } from '@material-ui/core';
const useStyles = makeStyles(() => ({
root: {
@@ -32,14 +30,19 @@ const useStyles = makeStyles(() => ({
}));
type Props = {
- theme?: PageTheme;
+ themeId: string;
};
-export const Page: FC = ({ theme = pageTheme.home, children }) => {
+export const Page: FC = ({ themeId, children }) => {
const classes = useStyles();
return (
-
+ ({
+ ...baseTheme,
+ page: baseTheme.getPageTheme({ themeId }),
+ })}
+ >
{children}
-
+
);
};
diff --git a/packages/core/src/layout/Page/index.ts b/packages/core/src/layout/Page/index.ts
index 000a565c83..987aee5cdb 100644
--- a/packages/core/src/layout/Page/index.ts
+++ b/packages/core/src/layout/Page/index.ts
@@ -15,5 +15,3 @@
*/
export { Page } from './Page';
-export { pageTheme } from './PageThemeProvider';
-export type { PageTheme } from './PageThemeProvider';
diff --git a/packages/core/src/layout/Sidebar/DefaultProviderSettings.tsx b/packages/core/src/layout/Sidebar/DefaultProviderSettings.tsx
deleted file mode 100644
index 99c5225c83..0000000000
--- a/packages/core/src/layout/Sidebar/DefaultProviderSettings.tsx
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import {
- configApiRef,
- githubAuthApiRef,
- gitlabAuthApiRef,
- googleAuthApiRef,
- oauth2ApiRef,
- oktaAuthApiRef,
- microsoftAuthApiRef,
- samlAuthApiRef,
- useApi,
-} from '@backstage/core-api';
-import Star from '@material-ui/icons/Star';
-import React from 'react';
-import { ProviderSettingsItem } from './Settings';
-
-export const DefaultProviderSettings = () => {
- const configApi = useApi(configApiRef);
- const providersConfig = configApi.getOptionalConfig('auth.providers');
- const providers = providersConfig?.keys() ?? [];
-
- return (
- <>
- {providers.includes('google') && (
- (theme => {
return {
root: {
- color: '#b5b5b5',
+ color: theme.palette.navigation.color,
display: 'flex',
flexFlow: 'row nowrap',
alignItems: 'center',
@@ -96,7 +103,7 @@ const useStyles = makeStyles(theme => {
selected: {
'&$root': {
borderLeft: `solid ${selectedIndicatorWidth}px ${theme.palette.navigation.indicator}`,
- color: '#ffffff',
+ color: theme.palette.navigation.selectedColor,
},
'&$closed': {
width: drawerWidthClosed - selectedIndicatorWidth,
@@ -115,74 +122,93 @@ type SidebarItemProps = {
to?: string;
hasNotifications?: boolean;
onClick?: () => void;
+ children?: ReactNode;
};
-export const SidebarItem: FC = ({
- icon: Icon,
- text,
- to,
- hasNotifications = false,
- onClick,
- children,
-}) => {
- const classes = useStyles();
- // XXX (@koroeskohr): unsure this is optimal. But I just really didn't want to have the item component
- // depend on the current location, and at least have it being optionally forced to selected.
- // Still waiting on a Q answered to fine tune the implementation
- const { isOpen } = useContext(SidebarContext);
+export const SidebarItem = forwardRef(
+ (
+ { icon: Icon, text, to, hasNotifications = false, onClick, children },
+ ref,
+ ) => {
+ const classes = useStyles();
+ // XXX (@koroeskohr): unsure this is optimal. But I just really didn't want to have the item component
+ // depend on the current location, and at least have it being optionally forced to selected.
+ // Still waiting on a Q answered to fine tune the implementation
+ const { isOpen } = useContext(SidebarContext);
- const itemIcon = (
-
-
- );
+ const itemIcon = (
+
+
+ );
- const childProps = {
- onClick,
- className: clsx(classes.root, isOpen ? classes.open : classes.closed),
- };
+ const childProps = {
+ onClick,
+ className: clsx(classes.root, isOpen ? classes.open : classes.closed),
+ };
+
+ if (!isOpen) {
+ if (to === undefined) {
+ return (
+
+ {itemIcon}
+
+ );
+ }
+
+ return (
+
+ {itemIcon}
+
+ );
+ }
+
+ const content = (
+ <>
+
+ {itemIcon}
+
+ {text && (
+
+ {text}
+
+ )}
+ {children}
+
+ );
- if (!isOpen) {
if (to === undefined) {
- return {itemIcon}
;
+ return (
+
+ {content}
+
+ );
}
return (
-
- {itemIcon}
+
+ {content}
);
- }
-
- const content = (
- <>
-
- {itemIcon}
-
- {text && (
-
- {text}
-
- )}
- {children}
-
- );
-
- if (to === undefined) {
- return {content}
;
- }
-
- return (
-
- {content}
-
- );
-};
+ },
+);
type SidebarSearchFieldProps = {
onSearch: (input: string) => void;
diff --git a/packages/core/src/layout/Sidebar/Page.tsx b/packages/core/src/layout/Sidebar/Page.tsx
index 2a4cf0fbb2..0c6f4a526d 100644
--- a/packages/core/src/layout/Sidebar/Page.tsx
+++ b/packages/core/src/layout/Sidebar/Page.tsx
@@ -45,7 +45,9 @@ export const SidebarPinStateContext = createContext(
);
export const SidebarPage: FC<{}> = props => {
- const [isPinned, setIsPinned] = useState(LocalStorage.getSidebarPinState());
+ const [isPinned, setIsPinned] = useState(() =>
+ LocalStorage.getSidebarPinState(),
+ );
useEffect(() => {
LocalStorage.setSidebarPinState(isPinned);
diff --git a/packages/core/src/layout/Sidebar/Settings/AppSettingsList.tsx b/packages/core/src/layout/Sidebar/Settings/AppSettingsList.tsx
deleted file mode 100644
index 5dad178ccd..0000000000
--- a/packages/core/src/layout/Sidebar/Settings/AppSettingsList.tsx
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import React from 'react';
-import { List, ListSubheader } from '@material-ui/core';
-import { SidebarThemeToggle } from './ThemeToggle';
-import { SidebarPinButton } from './PinButton';
-
-export const AppSettingsList = () => (
- App Settings}>
-
-);
diff --git a/packages/core/src/layout/Sidebar/Settings/FeatureFlagsItem.tsx b/packages/core/src/layout/Sidebar/Settings/FeatureFlagsItem.tsx
deleted file mode 100644
index 3e40ce69cc..0000000000
--- a/packages/core/src/layout/Sidebar/Settings/FeatureFlagsItem.tsx
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import {
- FeatureFlagName,
- useApi,
- featureFlagsApiRef,
-} from '@backstage/core-api';
-import {
- ListItem,
- ListItemSecondaryAction,
- ListItemText,
- Tooltip,
-} from '@material-ui/core';
-import CheckIcon from '@material-ui/icons/CheckCircle';
-import { ToggleButton } from '@material-ui/lab';
-
-export type Item = {
- name: FeatureFlagName;
- pluginId: string;
-};
-
-type Props = {
- featureFlag: Item;
-};
-
-export const FlagItem = ({ featureFlag }: Props) => {
- const api = useApi(featureFlagsApiRef);
-
- const [enabled, setEnabled] = React.useState(
- Boolean(api.getFlags().get(featureFlag.name)),
- );
-
- const toggleFlag = () => {
- const newState = api.getFlags().toggle(featureFlag.name);
- setEnabled(Boolean(newState));
- };
-
- return (
-
-
-
-
-
-
-
-
- );
-};
diff --git a/packages/core/src/layout/Sidebar/Settings/SettingsDialog.tsx b/packages/core/src/layout/Sidebar/Settings/SettingsDialog.tsx
deleted file mode 100644
index 71c4899862..0000000000
--- a/packages/core/src/layout/Sidebar/Settings/SettingsDialog.tsx
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import {
- Card,
- CardContent,
- CardHeader,
- makeStyles,
- Divider,
-} from '@material-ui/core';
-import { AppSettingsList } from './AppSettingsList';
-import { AuthProvidersList } from './AuthProviderList';
-import { FeatureFlagsList } from './FeatureFlagsList';
-import { SignInAvatar } from './SignInAvatar';
-import { UserSettingsMenu } from './UserSettingsMenu';
-import { useUserProfile } from './useUserProfileInfo';
-import { useApi, featureFlagsApiRef } from '@backstage/core-api';
-
-const useStyles = makeStyles({
- root: {
- minWidth: 400,
- },
-});
-
-type Props = {
- providerSettings?: React.ReactNode;
-};
-
-export const SettingsDialog = ({ providerSettings }: Props) => {
- const classes = useStyles();
- const { profile, displayName } = useUserProfile();
- const featureFlagsApi = useApi(featureFlagsApiRef);
- const featureFlags = featureFlagsApi.getRegisteredFlags();
-
- return (
-
-
-
-
- );
-};
diff --git a/packages/core/src/layout/Sidebar/Settings/ThemeToggle.tsx b/packages/core/src/layout/Sidebar/Settings/ThemeToggle.tsx
deleted file mode 100644
index a5e703089c..0000000000
--- a/packages/core/src/layout/Sidebar/Settings/ThemeToggle.tsx
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React from 'react';
-import { useObservable } from 'react-use';
-import LightIcon from '@material-ui/icons/WbSunny';
-import DarkIcon from '@material-ui/icons/Brightness2';
-import AutoIcon from '@material-ui/icons/BrightnessAuto';
-import { appThemeApiRef, useApi } from '@backstage/core-api';
-import ToggleButton from '@material-ui/lab/ToggleButton';
-import ToggleButtonGroup from '@material-ui/lab/ToggleButtonGroup';
-import {
- ListItem,
- ListItemText,
- ListItemSecondaryAction,
- Tooltip,
-} from '@material-ui/core';
-
-export const SidebarThemeToggle = () => {
- const appThemeApi = useApi(appThemeApiRef);
- const themeId = useObservable(
- appThemeApi.activeThemeId$(),
- appThemeApi.getActiveThemeId(),
- );
-
- const themeIds = appThemeApi.getInstalledThemes();
- // TODO(marcuseide): can these be put on the theme itself?
- const themeIcons = {
- dark: ,
- newThemeId: string | undefined,
- ) => {
- if (themeIds.some(t => t.id === newThemeId)) {
- appThemeApi.setActiveThemeId(newThemeId);
- } else {
- appThemeApi.setActiveThemeId(undefined);
- }
- };
-
- return (
-
-
-
- {themeIds.map(theme => (
-
-
- {themeIcons[theme.variant]}
-
-
- ))}
-
-
-
-
-
-
-
- );
-};
diff --git a/packages/core/src/layout/Sidebar/Settings/UserSettings.tsx b/packages/core/src/layout/Sidebar/Settings/UserSettings.tsx
deleted file mode 100644
index 85dbefff9b..0000000000
--- a/packages/core/src/layout/Sidebar/Settings/UserSettings.tsx
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import React, { useEffect, useContext } from 'react';
-import { Popover } from '@material-ui/core';
-import { SignInAvatar } from './SignInAvatar';
-import { SettingsDialog } from './SettingsDialog';
-import { SidebarItem } from '../Items';
-import { useUserProfile } from './useUserProfileInfo';
-import { SidebarContext } from '../config';
-
-type Props = {
- providerSettings?: React.ReactNode;
-};
-
-export const SidebarUserSettings = ({ providerSettings }: Props) => {
- const { isOpen: sidebarOpen } = useContext(SidebarContext);
- const { displayName } = useUserProfile();
- const [open, setOpen] = React.useState(false);
- const [anchorEl, setAnchorEl] = React.useState(
- undefined,
- );
-
- const handleOpen = (event?: React.MouseEvent) => {
- setAnchorEl(event?.currentTarget ?? undefined);
- setOpen(true);
- };
-
- const handleClose = () => {
- setAnchorEl(undefined);
- setOpen(false);
- };
-
- useEffect(() => {
- if (!sidebarOpen && open) setOpen(false);
- }, [open, sidebarOpen]);
-
- const SidebarAvatar = () =>
-
-
- );
-};
diff --git a/packages/core/src/layout/Sidebar/Sidebar.stories.tsx b/packages/core/src/layout/Sidebar/Sidebar.stories.tsx
index 61975a728a..d6a01fea33 100644
--- a/packages/core/src/layout/Sidebar/Sidebar.stories.tsx
+++ b/packages/core/src/layout/Sidebar/Sidebar.stories.tsx
@@ -22,17 +22,13 @@ import {
SidebarDivider,
SidebarSearchField,
SidebarSpace,
- SidebarUserSettings,
- ProviderSettingsItem,
} from '.';
import HomeOutlinedIcon from '@material-ui/icons/HomeOutlined';
import AddCircleOutlineIcon from '@material-ui/icons/AddCircleOutline';
-import Star from '@material-ui/icons/Star';
import { MemoryRouter } from 'react-router-dom';
-import { githubAuthApiRef } from '@backstage/core-api';
export default {
- title: 'Sidebar',
+ title: 'Layout/Sidebar',
component: Sidebar,
decorators: [
(storyFn: () => JSX.Element) => (
@@ -48,7 +44,6 @@ const handleSearch = (input: string) => {
export const SampleSidebar = () => (
- {/*
);
diff --git a/packages/core/src/layout/Sidebar/index.ts b/packages/core/src/layout/Sidebar/index.ts
index a644c06e14..803306478d 100644
--- a/packages/core/src/layout/Sidebar/index.ts
+++ b/packages/core/src/layout/Sidebar/index.ts
@@ -31,5 +31,3 @@ export {
sidebarConfig,
} from './config';
export type { SidebarContextType } from './config';
-export { DefaultProviderSettings } from './DefaultProviderSettings';
-export * from './Settings';
diff --git a/packages/core/src/layout/SignInPage/SignInPage.tsx b/packages/core/src/layout/SignInPage/SignInPage.tsx
index 8a1ff68250..ebc9bf278f 100644
--- a/packages/core/src/layout/SignInPage/SignInPage.tsx
+++ b/packages/core/src/layout/SignInPage/SignInPage.tsx
@@ -52,7 +52,7 @@ export const SignInPage: FC = ({
}
return (
-
+
{title &&
- Test Content
- Test Content
- ,
- ),
+ it('renders without exploding', async () => {
+ const rendered = await renderInTestApp(
+
+ Test Content
+ Test Content
+ ,
);
expect(rendered.getByText('Some title')).toBeInTheDocument();
});
- it('renders a deepLink when prop is set', () => {
- const rendered = render(
- wrapInTestApp(
-
- Test Content
- Test Content
- ,
- ),
+ it('renders a deepLink when prop is set', async () => {
+ const rendered = await renderInTestApp(
+
+ Test Content
+ Test Content
+ ,
);
expect(rendered.getByText('A deepLink title')).toBeInTheDocument();
});
- it('switches tabs when clicking', () => {
- const rendered = render(
- wrapInTestApp(
-
- Test Content 1
- Test Content 2
- ,
- ),
+ it('switches tabs when clicking', async () => {
+ const rendered = await renderInTestApp(
+
+ Test Content 1
+ Test Content 2
+ ,
);
expect(rendered.getByText('Test Content 1')).toBeInTheDocument();
diff --git a/packages/core/src/setupTests.ts b/packages/core/src/setupTests.ts
index 8553642152..825bcd4115 100644
--- a/packages/core/src/setupTests.ts
+++ b/packages/core/src/setupTests.ts
@@ -15,5 +15,3 @@
*/
import '@testing-library/jest-dom';
-
-require('jest-fetch-mock').enableMocks();
diff --git a/packages/create-app/bin/backstage-create-app b/packages/create-app/bin/backstage-create-app
index a065b8ac5a..467a0cae8f 100755
--- a/packages/create-app/bin/backstage-create-app
+++ b/packages/create-app/bin/backstage-create-app
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+/* eslint-disable no-restricted-syntax */
const path = require('path');
// Figure out whether we're running inside the backstage repo or as an installed dependency
diff --git a/packages/create-app/package.json b/packages/create-app/package.json
index 4ff0baf4fb..ff9e1fa636 100644
--- a/packages/create-app/package.json
+++ b/packages/create-app/package.json
@@ -1,7 +1,7 @@
{
"name": "@backstage/create-app",
"description": "Create app package for Backstage",
- "version": "0.1.1-alpha.24",
+ "version": "0.1.1-alpha.26",
"private": false,
"publishConfig": {
"access": "public"
@@ -27,7 +27,7 @@
"start": "nodemon --"
},
"dependencies": {
- "@backstage/cli-common": "^0.1.1-alpha.24",
+ "@backstage/cli-common": "^0.1.1-alpha.26",
"chalk": "^4.0.0",
"commander": "^6.1.0",
"fs-extra": "^9.0.0",
diff --git a/packages/create-app/src/createApp.ts b/packages/create-app/src/createApp.ts
index 28e3805ce8..83e052c91b 100644
--- a/packages/create-app/src/createApp.ts
+++ b/packages/create-app/src/createApp.ts
@@ -74,7 +74,6 @@ async function buildApp(appDir: string) {
await runCmd('yarn install');
await runCmd('yarn tsc');
- await runCmd('yarn build');
}
async function moveApp(tempDir: string, destination: string, id: string) {
@@ -88,6 +87,7 @@ async function moveApp(tempDir: string, destination: string, id: string) {
}
export default async (cmd: Command): Promise => {
+ /* eslint-disable-next-line no-restricted-syntax */
const paths = findPaths(__dirname);
const questions: Question[] = [
@@ -148,6 +148,10 @@ export default async (cmd: Command): Promise => {
chalk.green(`🥇 Successfully created ${chalk.cyan(answers.name)}`),
);
Task.log();
+ Task.log(
+ 'See https://backstage.io/docs/tutorials/quickstart-app-auth to know more about enabling auth providers',
+ );
+ Task.log();
Task.exit();
} catch (error) {
Task.error(error.message);
diff --git a/packages/create-app/templates/default-app/app-config.development.yaml b/packages/create-app/templates/default-app/app-config.development.yaml
deleted file mode 100644
index 817847c6d6..0000000000
--- a/packages/create-app/templates/default-app/app-config.development.yaml
+++ /dev/null
@@ -1,13 +0,0 @@
-app:
- baseUrl: http://localhost:3000
-
-backend:
- baseUrl: http://localhost:7000
- listen:
- port: 7000
- cors:
- origin: http://localhost:3000
- methods: [GET, POST, PUT, DELETE]
- credentials: true
- csp:
- connect-src: ["'self'", 'http:', 'https:']
diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs
index fb6185d51b..d643c72e0c 100644
--- a/packages/create-app/templates/default-app/app-config.yaml.hbs
+++ b/packages/create-app/templates/default-app/app-config.yaml.hbs
@@ -1,16 +1,20 @@
app:
title: Scaffolded Backstage App
- baseUrl: http://localhost:7000
+ baseUrl: http://localhost:3000
organization:
- name: Acme Corporation
+ name: My Company
backend:
baseUrl: http://localhost:7000
listen:
port: 7000
csp:
- connect-src: ["'self'", 'https:']
+ connect-src: ["'self'", 'http:', 'https:']
+ cors:
+ origin: http://localhost:3000
+ methods: [GET, POST, PUT, DELETE]
+ credentials: true
{{#if dbTypeSqlite}}
database:
client: sqlite3
@@ -34,6 +38,18 @@ backend:
#ca: # if you have a CA file and want to verify it you can uncomment this section
# $file: /ca/server.crt
{{/if}}
+ # workingDirectory: /tmp # Use this to configure a working direcotry for the scaffolder, defaults to the OS temp-dir
+
+integrations:
+ github:
+ - host: github.com
+ token:
+ $env: GITHUB_TOKEN
+ ### Example for how to add your GitHub Enterprise instance using the API:
+ # - host: ghe.example.net
+ # apiBaseUrl: https://ghe.example.net/api/v3
+ # token:
+ # $env: GHE_TOKEN
proxy:
'/test':
@@ -42,7 +58,7 @@ proxy:
techdocs:
storageUrl: http://localhost:7000/api/techdocs/static/docs
- requestUrl: http://localhost:7000/api/techdocs/docs
+ requestUrl: http://localhost:7000/api/techdocs
generators:
techdocs: 'docker'
@@ -50,28 +66,18 @@ lighthouse:
baseUrl: http://localhost:3003
auth:
+ # see https://backstage.io/docs/tutorials/quickstart-app-auth to know more about enabling auth providers
providers: {}
scaffolder:
github:
token:
- $env: GITHUB_ACCESS_TOKEN
+ $env: GITHUB_TOKEN
visibility: public # or 'internal' or 'private'
catalog:
rules:
- allow: [Component, API, Group, User, Template, Location]
- processors:
- github:
- providers:
- - target: https://github.com
- token:
- $env: GITHUB_PRIVATE_TOKEN
- # Example for how to add your GitHub Enterprise instance:
- # - target: https://ghe.example.net
- # apiBaseUrl: https://ghe.example.net/api/v3
- # token:
- # $env: GHE_PRIVATE_TOKEN
locations:
# Backstage example components
- type: url
@@ -82,23 +88,23 @@ catalog:
target: https://github.com/spotify/backstage/blob/master/packages/catalog-model/examples/all-apis.yaml
# Backstage example templates
- - type: url
+ - type: github
target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/react-ssr-template/template.yaml
rules:
- allow: [Template]
- - type: url
+ - type: github
target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/springboot-grpc-template/template.yaml
rules:
- allow: [Template]
- - type: url
+ - type: github
target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/create-react-app/template.yaml
rules:
- allow: [Template]
- - type: url
+ - type: github
target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml
rules:
- allow: [Template]
- - type: url
+ - type: github
target: https://github.com/spotify/backstage/blob/master/plugins/scaffolder-backend/sample-templates/docs-template/template.yaml
rules:
- allow: [Template]
diff --git a/packages/create-app/templates/default-app/packages/app/cypress/integration/app.js b/packages/create-app/templates/default-app/packages/app/cypress/integration/app.js
index dde26b7b39..efcd5b8d93 100644
--- a/packages/create-app/templates/default-app/packages/app/cypress/integration/app.js
+++ b/packages/create-app/templates/default-app/packages/app/cypress/integration/app.js
@@ -1,6 +1,6 @@
describe('App', () => {
it('should render the catalog', () => {
cy.visit('/');
- cy.contains('Backstage Service Catalog');
+ cy.contains('My Company Service Catalog');
});
});
diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs
index a057826ba4..103e75f6e0 100644
--- a/packages/create-app/templates/default-app/packages/app/package.json.hbs
+++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs
@@ -19,6 +19,7 @@
"@backstage/plugin-lighthouse": "^{{version}}",
"@backstage/plugin-tech-radar": "^{{version}}",
"@backstage/plugin-github-actions": "^{{version}}",
+ "@backstage/plugin-user-settings": "^{{version}}",
"@backstage/test-utils": "^{{version}}",
"@backstage/theme": "^{{version}}",
"history": "^5.0.0",
diff --git a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx
index 5480d8218a..7d65b3261a 100644
--- a/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx
+++ b/packages/create-app/templates/default-app/packages/app/src/components/catalog/EntityPage.tsx
@@ -53,9 +53,9 @@ const CICDSwitcher = ({ entity }: { entity: Entity }) => {
};
const OverviewContent = ({ entity }: { entity: Entity }) => (
-
+
-
);
@@ -77,7 +77,7 @@ const ServiceEntityPage = ({ entity }: { entity: Entity }) => (
title="API"
element={ = {
wave: `url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='1368' height='400' fill='none'%3e%3cmask id='a' width='1368' height='401' x='0' y='0' maskUnits='userSpaceOnUse'%3e%3cpath fill='url(%23paint0_linear)' d='M437 116C223 116 112 0 112 0h1256v400c-82 0-225-21-282-109-112-175-436-175-649-175z'/%3e%3cpath fill='url(%23paint1_linear)' d='M1368 400V282C891-29 788 40 711 161 608 324 121 372 0 361v39h1368z'/%3e%3cpath fill='url(%23paint2_linear)' d='M1368 244v156H0V94c92-24 198-46 375 0l135 41c176 51 195 109 858 109z'/%3e%3cpath fill='url(%23paint3_linear)' d='M1252 400h116c-14-7-35-14-116-16-663-14-837-128-1013-258l-85-61C98 28 46 8 0 0v400h1252z'/%3e%3c/mask%3e%3cg mask='url(%23a)'%3e%3cpath fill='white' d='M-172-98h1671v601H-172z'/%3e%3c/g%3e%3cdefs%3e%3clinearGradient id='paint0_linear' x1='602' x2='1093.5' y1='-960.5' y2='272' gradientUnits='userSpaceOnUse'%3e%3cstop stop-color='white'/%3e%3cstop offset='1' stop-color='white' stop-opacity='0'/%3e%3c/linearGradient%3e%3clinearGradient id='paint1_linear' x1='482' x2='480' y1='1058.5' y2='70.5' gradientUnits='userSpaceOnUse'%3e%3cstop stop-color='white'/%3e%3cstop offset='1' stop-color='white' stop-opacity='0'/%3e%3c/linearGradient%3e%3clinearGradient id='paint2_linear' x1='424' x2='446.1' y1='-587.5' y2='274.6' gradientUnits='userSpaceOnUse'%3e%3cstop stop-color='white'/%3e%3cstop offset='1' stop-color='white' stop-opacity='0'/%3e%3c/linearGradient%3e%3clinearGradient id='paint3_linear' x1='587' x2='349' y1='-1120.5' y2='341' gradientUnits='userSpaceOnUse'%3e%3cstop stop-color='white'/%3e%3cstop offset='1' stop-color='white' stop-opacity='0'/%3e%3c/linearGradient%3e%3c/defs%3e%3c/svg%3e")`,
@@ -49,24 +43,24 @@ export const colorVariants: Record = {
pinkSea: ['#C8077A', '#C2297D'],
};
-export const pageTheme: Record = {
- home: genTheme(colorVariants.teal, shapes.wave),
- documentation: genTheme(colorVariants.pinkSea, shapes.wave2),
- tool: genTheme(colorVariants.purpleSky, shapes.round),
- service: genTheme(colorVariants.marineBlue, shapes.wave),
- website: genTheme(colorVariants.veryBlue, shapes.wave),
- library: genTheme(colorVariants.rubyRed, shapes.wave),
- other: genTheme(colorVariants.darkGrey, shapes.wave),
- app: genTheme(colorVariants.toastyOrange, shapes.wave),
-};
-
// As the background shapes and colors are decorative, we place them onto
// the page as a css background-image instead of an html element of its own.
// Utility to not have to write colors and shapes twice.
-function genTheme(colors: string[], shape: string) {
+export function genPageTheme(colors: string[], shape: string): PageTheme {
const gradientColors = colors.length === 1 ? [colors[0], colors[0]] : colors;
const gradient = `linear-gradient(90deg, ${gradientColors.join(', ')})`;
const backgroundImage = `${shape}, ${gradient}`;
return { colors, shape, backgroundImage };
}
+
+export const pageTheme: Record = {
+ home: genPageTheme(colorVariants.teal, shapes.wave),
+ documentation: genPageTheme(colorVariants.pinkSea, shapes.wave2),
+ tool: genPageTheme(colorVariants.purpleSky, shapes.round),
+ service: genPageTheme(colorVariants.marineBlue, shapes.wave),
+ website: genPageTheme(colorVariants.veryBlue, shapes.wave),
+ library: genPageTheme(colorVariants.rubyRed, shapes.wave),
+ other: genPageTheme(colorVariants.darkGrey, shapes.wave),
+ app: genPageTheme(colorVariants.toastyOrange, shapes.wave),
+};
diff --git a/packages/theme/src/themes.ts b/packages/theme/src/themes.ts
index 0ec3304ac5..5b233ac303 100644
--- a/packages/theme/src/themes.ts
+++ b/packages/theme/src/themes.ts
@@ -15,6 +15,7 @@
*/
import { createTheme } from './baseTheme';
+import { pageTheme } from './pageTheme';
import { yellow } from '@material-ui/core/colors';
export const lightTheme = createTheme({
@@ -64,6 +65,8 @@ export const lightTheme = createTheme({
navigation: {
background: '#171717',
indicator: '#9BF0E1',
+ color: '#b5b5b5',
+ selectedColor: '#FFF',
},
pinSidebarButton: {
icon: '#181818',
@@ -73,6 +76,8 @@ export const lightTheme = createTheme({
indicator: '#9BF0E1',
},
},
+ defaultPageTheme: 'home',
+ pageTheme,
});
export const darkTheme = createTheme({
@@ -122,6 +127,8 @@ export const darkTheme = createTheme({
navigation: {
background: '#424242',
indicator: '#9BF0E1',
+ color: '#b5b5b5',
+ selectedColor: '#FFF',
},
pinSidebarButton: {
icon: '#404040',
@@ -131,4 +138,6 @@ export const darkTheme = createTheme({
indicator: '#9BF0E1',
},
},
+ defaultPageTheme: 'home',
+ pageTheme,
});
diff --git a/packages/theme/src/types.ts b/packages/theme/src/types.ts
index 1d244f8cd4..5acc0f75e9 100644
--- a/packages/theme/src/types.ts
+++ b/packages/theme/src/types.ts
@@ -46,6 +46,8 @@ type PaletteAdditions = {
navigation: {
background: string;
indicator: string;
+ color: string;
+ selectedColor: string;
};
tabbar: {
indicator: string;
@@ -72,12 +74,20 @@ type PaletteAdditions = {
export type BackstagePalette = Palette & PaletteAdditions;
export type BackstagePaletteOptions = PaletteOptions & PaletteAdditions;
+export type PageThemeSelector = {
+ themeId: string;
+};
+
export interface BackstageTheme extends Theme {
palette: BackstagePalette;
+ page: PageTheme;
+ getPageTheme: ({ themeId }: PageThemeSelector) => PageTheme;
}
export interface BackstageThemeOptions extends ThemeOptions {
palette: BackstagePaletteOptions;
+ page: PageTheme;
+ getPageTheme: ({ themeId }: PageThemeSelector) => PageTheme;
}
/**
@@ -85,5 +95,13 @@ export interface BackstageThemeOptions extends ThemeOptions {
*/
export type SimpleThemeOptions = {
palette: BackstagePaletteOptions;
+ defaultPageTheme: string;
+ pageTheme?: Record;
fontFamily?: string;
};
+
+export type PageTheme = {
+ colors: string[];
+ shape: string;
+ backgroundImage: string;
+};
diff --git a/plugins/api-docs/package.json b/plugins/api-docs/package.json
index a703b6afe5..baf574d5b5 100644
--- a/plugins/api-docs/package.json
+++ b/plugins/api-docs/package.json
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-api-docs",
- "version": "0.1.1-alpha.24",
+ "version": "0.1.1-alpha.26",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
@@ -20,10 +20,10 @@
"clean": "backstage-cli clean"
},
"dependencies": {
- "@backstage/catalog-model": "^0.1.1-alpha.24",
- "@backstage/core": "^0.1.1-alpha.24",
- "@backstage/plugin-catalog": "^0.1.1-alpha.24",
- "@backstage/theme": "^0.1.1-alpha.24",
+ "@backstage/catalog-model": "^0.1.1-alpha.26",
+ "@backstage/core": "^0.1.1-alpha.26",
+ "@backstage/plugin-catalog": "^0.1.1-alpha.26",
+ "@backstage/theme": "^0.1.1-alpha.26",
"@kyma-project/asyncapi-react": "^0.13.1",
"@material-icons/font": "^1.0.2",
"@material-ui/core": "^4.11.0",
@@ -39,9 +39,9 @@
"swagger-ui-react": "^3.31.1"
},
"devDependencies": {
- "@backstage/cli": "^0.1.1-alpha.24",
- "@backstage/dev-utils": "^0.1.1-alpha.24",
- "@backstage/test-utils": "^0.1.1-alpha.24",
+ "@backstage/cli": "^0.1.1-alpha.26",
+ "@backstage/dev-utils": "^0.1.1-alpha.26",
+ "@backstage/test-utils": "^0.1.1-alpha.26",
"@testing-library/jest-dom": "^5.10.1",
"@testing-library/react": "^10.4.1",
"@testing-library/user-event": "^12.0.7",
@@ -49,9 +49,8 @@
"@types/node": "^12.0.0",
"@types/react": "^16.9",
"@types/swagger-ui-react": "^3.23.3",
- "jest-fetch-mock": "^3.0.3",
- "msw": "^0.20.5",
- "node-fetch": "^2.6.1"
+ "cross-fetch": "^3.0.6",
+ "msw": "^0.21.2"
},
"files": [
"dist"
diff --git a/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/MissingImplementsApisEmptyState.tsx b/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/MissingImplementsApisEmptyState.tsx
new file mode 100644
index 0000000000..9160aca514
--- /dev/null
+++ b/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/MissingImplementsApisEmptyState.tsx
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { Button, makeStyles, Typography } from '@material-ui/core';
+import { BackstageTheme } from '@backstage/theme';
+import { CodeSnippet, EmptyState } from '@backstage/core';
+
+const COMPONENT_YAML = `# Example
+apiVersion: backstage.io/v1alpha1
+kind: Component
+metadata:
+ name: example
+spec:
+ type: service
+ lifecycle: production
+ owner: guest
+ implementsApis:
+ - example-api
+`;
+
+const useStyles = makeStyles(theme => ({
+ code: {
+ borderRadius: 6,
+ margin: `${theme.spacing(2)}px 0px`,
+ background: theme.palette.type === 'dark' ? '#444' : '#fff',
+ },
+}));
+
+export const MissingImplementsApisEmptyState = () => {
+ const classes = useStyles();
+ return (
+
+ Components can implement APIs that are displayed on this page. You
+ need to fill the implementsApis field to enable this
+ tool.
+
+ }
+ action={
+ <>
+
+ Link an API to your component as shown in the highlighted example
+ below:
+
+
+
+
+ Read more
+
+
+ }
+ />
+ );
+};
diff --git a/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/index.ts b/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/index.ts
new file mode 100644
index 0000000000..1b7d35c0a2
--- /dev/null
+++ b/plugins/api-docs/src/catalog/MissingImplementsApisEmptyState/index.ts
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { MissingImplementsApisEmptyState } from './MissingImplementsApisEmptyState';
diff --git a/plugins/api-docs/src/catalog/Router.tsx b/plugins/api-docs/src/catalog/Router.tsx
index 71640954b3..d3fe7f7553 100644
--- a/plugins/api-docs/src/catalog/Router.tsx
+++ b/plugins/api-docs/src/catalog/Router.tsx
@@ -17,20 +17,17 @@
import React from 'react';
import { Entity } from '@backstage/catalog-model';
import { Route, Routes } from 'react-router';
-import { WarningPanel } from '@backstage/core';
import { catalogRoute } from '../routes';
import { EntityPageApi } from './EntityPageApi';
+import { MissingImplementsApisEmptyState } from './MissingImplementsApisEmptyState';
const isPluginApplicableToEntity = (entity: Entity) => {
return ((entity.spec?.implementsApis as string[]) || []).length > 0;
};
export const Router = ({ entity }: { entity: Entity }) =>
- // TODO(shmidt-i): move warning to a separate standardized component
!isPluginApplicableToEntity(entity) ? (
-
- The entity doesn't implement any APIs.
-
+
{
const actual = jest.requireActual('react-router-dom');
@@ -71,32 +69,3 @@ describe('ApiEntityPage', () => {
);
});
});
-
-describe('getPageTheme', () => {
- const defaultPageTheme = getPageTheme();
- it.each(['service', 'app', 'library', 'tool', 'documentation', 'website'])(
- 'should select right theme for predefined type: %p ̰ ',
- type => {
- const theme = getPageTheme(({
- spec: {
- type,
- },
- } as any) as Entity);
- expect(theme).toBeDefined();
- expect(theme).not.toBe(defaultPageTheme);
- },
- );
-
- it('should select default theme for unknown/unspecified types', () => {
- const theme1 = getPageTheme(({
- spec: {
- type: 'unknown-type',
- },
- } as any) as Entity);
- const theme2 = getPageTheme(({
- spec: {},
- } as any) as Entity);
- expect(theme1).toBe(defaultPageTheme);
- expect(theme2).toBe(defaultPageTheme);
- });
-});
diff --git a/plugins/api-docs/src/components/ApiEntityPage/ApiEntityPage.tsx b/plugins/api-docs/src/components/ApiEntityPage/ApiEntityPage.tsx
index b7e9996cc0..f755705ac7 100644
--- a/plugins/api-docs/src/components/ApiEntityPage/ApiEntityPage.tsx
+++ b/plugins/api-docs/src/components/ApiEntityPage/ApiEntityPage.tsx
@@ -20,8 +20,6 @@ import {
errorApiRef,
Header,
Page,
- pageTheme,
- PageTheme,
Progress,
useApi,
} from '@backstage/core';
@@ -54,11 +52,6 @@ function headerProps(
};
}
-export const getPageTheme = (entity?: Entity): PageTheme => {
- const themeKey = entity?.spec?.type?.toString() ?? 'home';
- return pageTheme[themeKey] ?? pageTheme.home;
-};
-
type EntityPageTitleProps = {
title: string;
entity: Entity | undefined;
@@ -107,7 +100,7 @@ export const ApiEntityPage = () => {
);
return (
-
+
}
pageTitleOverride={headerTitle}
diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerLayout.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerLayout.tsx
index a111007278..0720ecf45b 100644
--- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerLayout.tsx
+++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerLayout.tsx
@@ -14,22 +14,20 @@
* limitations under the License.
*/
-import { Header, Page, pageTheme } from '@backstage/core';
+import { Header, Page } from '@backstage/core';
import React from 'react';
type Props = {
children?: React.ReactNode;
};
-export const ApiExplorerLayout = ({ children }: Props) => {
- return (
-
-
- {children}
-
- );
-};
+export const ApiExplorerLayout = ({ children }: Props) => (
+
+
+ {children}
+
+);
diff --git a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx
index e386f281e8..46cf014134 100644
--- a/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx
+++ b/plugins/api-docs/src/components/ApiExplorerPage/ApiExplorerPage.test.tsx
@@ -72,6 +72,6 @@ describe('ApiCatalogPage', () => {
// https://github.com/mbrn/material-table/issues/1293
it('should render', async () => {
const { findByText } = renderWrapped(All your APIs
{
wrapInTestApp(
{
const rendered = render(
wrapInTestApp(
- ,
),
);
- expect(rendered.getByText(/APIs \(3\)/)).toBeInTheDocument();
expect(rendered.getByText(/api1/)).toBeInTheDocument();
expect(rendered.getByText(/api2/)).toBeInTheDocument();
expect(rendered.getByText(/api3/)).toBeInTheDocument();
diff --git a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx
index 485fcc216d..f3be9f664b 100644
--- a/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx
+++ b/plugins/api-docs/src/components/ApiExplorerTable/ApiExplorerTable.tsx
@@ -15,7 +15,7 @@
*/
import { ApiEntityV1alpha1, Entity } from '@backstage/catalog-model';
-import { Table, TableColumn, useApi } from '@backstage/core';
+import { Table, TableFilter, TableColumn, useApi } from '@backstage/core';
import { Chip, Link } from '@material-ui/core';
import { Alert } from '@material-ui/lab';
import React from 'react';
@@ -90,9 +90,27 @@ const columns: TableColumn[] = [
},
];
+const filters: TableFilter[] = [
+ {
+ column: 'Owner',
+ type: 'select',
+ },
+ {
+ column: 'Type',
+ type: 'multiple-select',
+ },
+ {
+ column: 'Lifecycle',
+ type: 'multiple-select',
+ },
+ {
+ column: 'Tags',
+ type: 'checkbox-tree',
+ },
+];
+
type ExplorerTableProps = {
entities: Entity[];
- titlePreamble: string;
loading: boolean;
error?: any;
};
@@ -101,7 +119,6 @@ export const ApiExplorerTable = ({
entities,
loading,
error,
- titlePreamble,
}: ExplorerTableProps) => {
if (error) {
return (
@@ -114,7 +131,7 @@ export const ApiExplorerTable = ({
}
return (
-
+
);
};
diff --git a/plugins/api-docs/src/components/useComponentApiEntities.ts b/plugins/api-docs/src/components/useComponentApiEntities.ts
index 11b5de988f..9e5cbd968e 100644
--- a/plugins/api-docs/src/components/useComponentApiEntities.ts
+++ b/plugins/api-docs/src/components/useComponentApiEntities.ts
@@ -16,7 +16,11 @@
import { useAsyncRetry } from 'react-use';
import { errorApiRef, useApi } from '@backstage/core';
-import { ApiEntity, ComponentEntity } from '@backstage/catalog-model';
+import {
+ ApiEntity,
+ ComponentEntity,
+ parseEntityName,
+} from '@backstage/catalog-model';
import { catalogApiRef } from '@backstage/plugin-catalog';
import { useComponentApiNames } from './useComponentApiNames';
@@ -43,10 +47,20 @@ export function useComponentApiEntities({
await Promise.all(
apiNames.map(async name => {
try {
- const api = (await catalogApi.getEntityByName({
- kind: 'API',
- name,
- })) as ApiEntity | undefined;
+ const apiEntityName = parseEntityName(name, {
+ defaultNamespace: entity.metadata.namespace,
+ defaultKind: 'API',
+ });
+
+ if (apiEntityName.kind !== 'API') {
+ throw new Error(
+ `Referenced entity of kind "${apiEntityName.kind}" as an API`,
+ );
+ }
+
+ const api = (await catalogApi.getEntityByName(apiEntityName)) as
+ | ApiEntity
+ | undefined;
if (api) {
resultMap.set(api.metadata.name, api);
diff --git a/plugins/api-docs/src/setupTests.ts b/plugins/api-docs/src/setupTests.ts
index 8553642152..825bcd4115 100644
--- a/plugins/api-docs/src/setupTests.ts
+++ b/plugins/api-docs/src/setupTests.ts
@@ -15,5 +15,3 @@
*/
import '@testing-library/jest-dom';
-
-require('jest-fetch-mock').enableMocks();
diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json
index 98873f88c7..8297b466af 100644
--- a/plugins/app-backend/package.json
+++ b/plugins/app-backend/package.json
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-app-backend",
- "version": "0.1.1-alpha.24",
+ "version": "0.1.1-alpha.26",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
@@ -20,8 +20,8 @@
"clean": "backstage-cli clean"
},
"dependencies": {
- "@backstage/backend-common": "^0.1.1-alpha.24",
- "@backstage/config-loader": "^0.1.1-alpha.24",
+ "@backstage/backend-common": "^0.1.1-alpha.26",
+ "@backstage/config-loader": "^0.1.1-alpha.26",
"@types/express": "^4.17.6",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
@@ -30,7 +30,7 @@
"yn": "^4.0.0"
},
"devDependencies": {
- "@backstage/cli": "^0.1.1-alpha.24",
+ "@backstage/cli": "^0.1.1-alpha.26",
"@types/supertest": "^2.0.8",
"msw": "^0.20.5",
"supertest": "^4.0.2"
diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts
index 417fdc43b6..aed4280ca5 100644
--- a/plugins/app-backend/src/service/router.test.ts
+++ b/plugins/app-backend/src/service/router.test.ts
@@ -25,6 +25,7 @@ import { createRouter } from './router';
jest.mock('../lib/config', () => ({ injectEnvConfig: jest.fn() }));
global.__non_webpack_require__ = {
+ /* eslint-disable-next-line no-restricted-syntax */
resolve: () => resolvePath(__dirname, '__fixtures__/app-dir/package.json'),
};
diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json
index fca8fb7178..6489c8547d 100644
--- a/plugins/auth-backend/package.json
+++ b/plugins/auth-backend/package.json
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-auth-backend",
- "version": "0.1.1-alpha.24",
+ "version": "0.1.1-alpha.26",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
@@ -20,13 +20,14 @@
"clean": "backstage-cli clean"
},
"dependencies": {
- "@backstage/backend-common": "^0.1.1-alpha.24",
- "@backstage/catalog-model": "^0.1.1-alpha.24",
- "@backstage/config": "^0.1.1-alpha.24",
+ "@backstage/backend-common": "^0.1.1-alpha.26",
+ "@backstage/catalog-model": "^0.1.1-alpha.26",
+ "@backstage/config": "^0.1.1-alpha.26",
"@types/express": "^4.17.6",
"compression": "^1.7.4",
"cookie-parser": "^1.4.5",
"cors": "^2.8.5",
+ "cross-fetch": "^3.0.6",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"fs-extra": "^9.0.0",
@@ -37,7 +38,6 @@
"knex": "^0.21.1",
"moment": "^2.26.0",
"morgan": "^1.10.0",
- "node-fetch": "^2.6.1",
"passport": "^0.4.1",
"passport-github2": "^0.1.12",
"passport-gitlab2": "^5.0.0",
@@ -51,18 +51,16 @@
"yn": "^4.0.0"
},
"devDependencies": {
- "@backstage/cli": "^0.1.1-alpha.24",
+ "@backstage/cli": "^0.1.1-alpha.26",
"@types/body-parser": "^1.19.0",
"@types/cookie-parser": "^1.4.2",
"@types/jwt-decode": "2.2.1",
- "@types/node-fetch": "^2.5.7",
"@types/passport": "^1.0.3",
"@types/passport-github2": "^1.2.4",
"@types/passport-google-oauth20": "^2.0.3",
"@types/passport-microsoft": "^0.0.0",
"@types/passport-saml": "^1.1.2",
- "jest-fetch-mock": "^3.0.3",
- "msw": "^0.20.5"
+ "msw": "^0.21.2"
},
"files": [
"dist",
diff --git a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts
index 2c1d66e1f8..e56763f656 100644
--- a/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts
+++ b/plugins/auth-backend/src/lib/catalog/CatalogIdentityClient.ts
@@ -14,13 +14,13 @@
* limitations under the License.
*/
-import fetch from 'node-fetch';
-import { UserEntity } from '@backstage/catalog-model';
import {
ConflictError,
NotFoundError,
PluginEndpointDiscovery,
} from '@backstage/backend-common';
+import { UserEntity } from '@backstage/catalog-model';
+import fetch from 'cross-fetch';
type UserQuery = {
annotations: Record;
@@ -42,15 +42,17 @@ export class CatalogIdentityClient {
* Throws a NotFoundError or ConflictError if 0 or multiple users are found.
*/
async findUser(query: UserQuery): Promise {
- const params = new URLSearchParams();
- params.append('kind', 'User');
-
+ const conditions = ['kind=user'];
for (const [key, value] of Object.entries(query.annotations)) {
- params.append(`metadata.annotations.${key}`, value);
+ const uk = encodeURIComponent(key);
+ const uv = encodeURIComponent(value);
+ conditions.push(`metadata.annotations.${uk}=${uv}`);
}
const baseUrl = await this.discovery.getBaseUrl('catalog');
- const response = await fetch(`${baseUrl}/entities?${params}`);
+ const response = await fetch(
+ `${baseUrl}/entities?filter=${conditions.join(',')}`,
+ );
if (!response.ok) {
const text = await response.text();
diff --git a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts
index d8f6d25d0f..f7a22e9ad9 100644
--- a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts
+++ b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.test.ts
@@ -16,6 +16,7 @@
import express from 'express';
import passport from 'passport';
+import { InternalOAuthError } from 'passport-oauth2';
import {
executeRedirectStrategy,
executeFrameHandlerStrategy,
@@ -58,7 +59,11 @@ describe('PassportStrategyHelper', () => {
}
class MyCustomAuthErrorStrategy extends passport.Strategy {
authenticate() {
- this.error(new Error('MyCustomAuth error'));
+ this.error(
+ new InternalOAuthError('MyCustomAuth error', {
+ data: '{ "message": "Custom message" }',
+ }),
+ );
}
}
class MyCustomAuthRedirectStrategy extends passport.Strategy {
@@ -97,7 +102,7 @@ describe('PassportStrategyHelper', () => {
);
expect(spyAuthenticate).toBeCalledTimes(1);
await expect(frameHandlerStrategyPromise).rejects.toThrow(
- 'Authentication failed, Error: MyCustomAuth error',
+ 'Authentication failed, MyCustomAuth error - Custom message',
);
});
diff --git a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts
index 7bc34186f4..3264fd8faa 100644
--- a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts
+++ b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts
@@ -18,6 +18,7 @@ import express from 'express';
import passport from 'passport';
import jwtDecoder from 'jwt-decode';
import { ProfileInfo, RedirectInfo } from '../../providers/types';
+import { InternalOAuthError } from 'passport-oauth2';
export type PassportDoneCallback = (
err?: Error,
@@ -95,8 +96,22 @@ export const executeFrameHandlerStrategy = async (
) => {
reject(new Error(`Authentication rejected, ${info.message ?? ''}`));
};
- strategy.error = (error: Error) => {
- reject(new Error(`Authentication failed, ${error}`));
+ strategy.error = (error: InternalOAuthError) => {
+ let message = `Authentication failed, ${error.message}`;
+
+ if (error.oauthError?.data) {
+ try {
+ const errorData = JSON.parse(error.oauthError.data);
+
+ if (errorData.message) {
+ message += ` - ${errorData.message}`;
+ }
+ } catch (parseError) {
+ message += ` - ${error.oauthError}`;
+ }
+ }
+
+ reject(new Error(message));
};
strategy.redirect = () => {
reject(new Error('Unexpected redirect'));
diff --git a/plugins/auth-backend/src/providers/saml/provider.ts b/plugins/auth-backend/src/providers/saml/provider.ts
index a90f3e3053..a8edc9a714 100644
--- a/plugins/auth-backend/src/providers/saml/provider.ts
+++ b/plugins/auth-backend/src/providers/saml/provider.ts
@@ -41,8 +41,10 @@ type SamlInfo = {
export class SamlAuthProvider implements AuthProviderRouteHandlers {
private readonly strategy: SamlStrategy;
private readonly tokenIssuer: TokenIssuer;
+ private readonly appUrl: string;
constructor(options: SAMLProviderOptions) {
+ this.appUrl = options.appUrl;
this.tokenIssuer = options.tokenIssuer;
this.strategy = new SamlStrategy({ ...options }, ((
profile: SamlProfile,
@@ -82,7 +84,7 @@ export class SamlAuthProvider implements AuthProviderRouteHandlers {
claims: { sub: id },
});
- return postMessageResponse(res, 'http://localhost:3000', {
+ return postMessageResponse(res, this.appUrl, {
type: 'authorization_response',
response: {
providerInfo: {},
@@ -91,7 +93,7 @@ export class SamlAuthProvider implements AuthProviderRouteHandlers {
},
});
} catch (error) {
- return postMessageResponse(res, 'http://localhost:3000', {
+ return postMessageResponse(res, this.appUrl, {
type: 'authorization_response',
error: {
name: error.name,
@@ -115,19 +117,24 @@ type SAMLProviderOptions = {
issuer: string;
path: string;
tokenIssuer: TokenIssuer;
+ appUrl: string;
};
export const createSamlProvider: AuthProviderFactory = ({
+ globalConfig,
config,
tokenIssuer,
}) => {
+ const url = new URL(globalConfig.baseUrl);
+ const providerId = 'saml';
const entryPoint = config.getString('entryPoint');
const issuer = config.getString('issuer');
const opts = {
entryPoint,
issuer,
- path: '/auth/saml/handler/frame',
+ path: `${url.pathname}/${providerId}/handler/frame`,
tokenIssuer,
+ appUrl: globalConfig.appUrl,
};
return new SamlAuthProvider(opts);
diff --git a/plugins/auth-backend/src/service/router.ts b/plugins/auth-backend/src/service/router.ts
index bcd1fc1794..6bb100de6b 100644
--- a/plugins/auth-backend/src/service/router.ts
+++ b/plugins/auth-backend/src/service/router.ts
@@ -17,7 +17,6 @@
import express from 'express';
import Router from 'express-promise-router';
import cookieParser from 'cookie-parser';
-import Knex from 'knex';
import { Logger } from 'winston';
import { createAuthProvider } from '../providers';
import { Config } from '@backstage/config';
@@ -25,11 +24,12 @@ import { DatabaseKeyStore, TokenFactory, createOidcRouter } from '../identity';
import {
NotFoundError,
PluginEndpointDiscovery,
+ PluginDatabaseManager,
} from '@backstage/backend-common';
export interface RouterOptions {
logger: Logger;
- database: Knex;
+ database: PluginDatabaseManager;
config: Config;
discovery: PluginEndpointDiscovery;
}
@@ -47,7 +47,9 @@ export async function createRouter({
const keyDurationSeconds = 3600;
- const keyStore = await DatabaseKeyStore.create({ database });
+ const keyStore = await DatabaseKeyStore.create({
+ database: await database.getClient(),
+ });
const tokenIssuer = new TokenFactory({
issuer: authUrl,
keyStore,
diff --git a/plugins/auth-backend/src/service/standaloneServer.ts b/plugins/auth-backend/src/service/standaloneServer.ts
index 71771340a5..a4f2377507 100644
--- a/plugins/auth-backend/src/service/standaloneServer.ts
+++ b/plugins/auth-backend/src/service/standaloneServer.ts
@@ -17,7 +17,6 @@
import Knex from 'knex';
import { Server } from 'http';
import { Logger } from 'winston';
-import { ConfigReader } from '@backstage/config';
import { createRouter } from './router';
import {
createServiceBuilder,
@@ -34,7 +33,7 @@ export async function startStandaloneServer(
options: ServerOptions,
): Promise {
const logger = options.logger.child({ service: 'auth-backend' });
- const config = ConfigReader.fromConfigs(await loadBackendConfig());
+ const config = await loadBackendConfig({ logger, argv: process.argv });
const discovery = SingleHostDiscovery.fromConfig(config);
const database = useHotMemoize(module, () => {
@@ -53,7 +52,11 @@ export async function startStandaloneServer(
const router = await createRouter({
logger,
config,
- database,
+ database: {
+ async getClient() {
+ return database;
+ },
+ },
discovery,
});
diff --git a/plugins/auth-backend/src/setupTests.ts b/plugins/auth-backend/src/setupTests.ts
index f7b6ca962d..ba33cf996b 100644
--- a/plugins/auth-backend/src/setupTests.ts
+++ b/plugins/auth-backend/src/setupTests.ts
@@ -14,6 +14,4 @@
* limitations under the License.
*/
-require('jest-fetch-mock').enableMocks();
-
export {};
diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js
new file mode 100644
index 0000000000..fdabf72fce
--- /dev/null
+++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+ await knex.schema.alterTable('entities', table => {
+ table
+ .text('data')
+ .nullable()
+ .comment('The entire JSON data blob of the entity');
+ });
+
+ await knex('entities').update({
+ // apiVersion and kind should not contain any JSON unsafe chars, and both
+ // metadata and spec are already valid serialized JSON
+ data: knex.raw(
+ `'{"apiVersion":"' || api_version || '","kind":"' || kind || '","metadata":' || metadata || COALESCE(',"spec":' || spec, '') || '}'`,
+ ),
+ });
+
+ await knex.schema.alterTable('entities', table => {
+ table.dropColumn('metadata');
+ table.dropColumn('spec');
+ });
+
+ // SQLite does not support ALTER COLUMN. Note that we do not use the try/
+ // catch method as in other migrations, because if the transaction is
+ // partially failed, it will further mess up the already messed-up
+ // statement below this.
+ if (knex.client.config.client !== 'sqlite3') {
+ await knex.schema.alterTable('entities', table => {
+ table.text('data').notNullable().alter();
+ });
+ }
+
+ // NOTE(freben): For some reason, specifically sqlite3 in-mem just drops some
+ // subset of constraints sometimes, when a table column is dropped - even if
+ // the column had no relation at all to the constraint. We therefore recreate
+ // the constraint here as a stupid fix.
+ if (knex.client.config.client === 'sqlite3') {
+ await knex.schema.alterTable('entities', table => {
+ table.unique(['full_name'], 'entities_unique_full_name');
+ });
+ }
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+ await knex.schema.alterTable('entities', table => {
+ table
+ .text('metadata')
+ .notNullable()
+ .comment('The entire metadata JSON blob of the entity');
+ table
+ .text('spec')
+ .nullable()
+ .comment('The entire spec JSON blob of the entity');
+ table.dropColumn('data');
+ });
+};
diff --git a/plugins/catalog-backend/migrations/20201006203131_entity_remove_redundant_columns.js b/plugins/catalog-backend/migrations/20201006203131_entity_remove_redundant_columns.js
new file mode 100644
index 0000000000..7bacc67338
--- /dev/null
+++ b/plugins/catalog-backend/migrations/20201006203131_entity_remove_redundant_columns.js
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+ await knex.schema.alterTable('entities', table => {
+ table.dropColumn('api_version');
+ table.dropColumn('kind');
+ table.dropColumn('name');
+ table.dropColumn('namespace');
+ });
+
+ // NOTE(freben): For some reason, specifically sqlite3 in-mem just drops some
+ // subset of constraints sometimes, when a table column is dropped - even if
+ // the column had no relation at all to the constraint. We therefore recreate
+ // the constraint here as a stupid fix.
+ if (knex.client.config.client === 'sqlite3') {
+ await knex.schema.alterTable('entities', table => {
+ table.unique(['full_name'], 'entities_unique_full_name');
+ });
+ }
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+ await knex.schema.alterTable('entities', table => {
+ table
+ .string('api_version')
+ .notNullable()
+ .comment('The apiVersion field of the entity');
+ table.string('kind').notNullable().comment('The kind field of the entity');
+ table
+ .string('name')
+ .nullable()
+ .comment('The metadata.name field of the entity');
+ table
+ .string('namespace')
+ .nullable()
+ .comment('The metadata.namespace field of the entity');
+ });
+};
diff --git a/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js
new file mode 100644
index 0000000000..b902073e64
--- /dev/null
+++ b/plugins/catalog-backend/migrations/20201007201501_index_entity_search.js
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+ await knex.schema.alterTable('entities_search', table => {
+ table.index(['key'], 'entities_search_key');
+ table.index(['value'], 'entities_search_value');
+ });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+ await knex.schema.alterTable('entities_search', table => {
+ table.dropIndex('', 'entities_search_key');
+ table.dropIndex('', 'entities_search_value');
+ });
+};
diff --git a/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js b/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js
new file mode 100644
index 0000000000..70150ff173
--- /dev/null
+++ b/plugins/catalog-backend/migrations/20201019130742_add_relations_table.js
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.up = async function up(knex) {
+ await knex.schema.createTable('entities_relations', table => {
+ table.comment('All relations between entities in the catalog');
+ table
+ .uuid('originating_entity_id')
+ .references('id')
+ .inTable('entities')
+ .onDelete('CASCADE')
+ .notNullable()
+ .comment('The entity that provided the relation');
+ table
+ .string('source_full_name')
+ .notNullable()
+ .comment('The full name of the source entity of the relation');
+ table
+ .string('type')
+ .notNullable()
+ .comment('The type of the relation between the entities');
+ table
+ .string('target_full_name')
+ .notNullable()
+ .comment('The full name of the target entity of the relation');
+
+ table.primary(['source_full_name', 'type', 'target_full_name']);
+ });
+};
+
+/**
+ * @param {import('knex')} knex
+ */
+exports.down = async function down(knex) {
+ await knex.schema.dropTable('entities_relations');
+};
diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json
index f16f629c44..cd8e363901 100644
--- a/plugins/catalog-backend/package.json
+++ b/plugins/catalog-backend/package.json
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-catalog-backend",
- "version": "0.1.1-alpha.24",
+ "version": "0.1.1-alpha.26",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
@@ -20,41 +20,41 @@
"clean": "backstage-cli clean"
},
"dependencies": {
- "@backstage/backend-common": "^0.1.1-alpha.24",
- "@backstage/catalog-model": "^0.1.1-alpha.24",
- "@backstage/config": "^0.1.1-alpha.24",
+ "@backstage/backend-common": "^0.1.1-alpha.26",
+ "@backstage/catalog-model": "^0.1.1-alpha.26",
+ "@backstage/config": "^0.1.1-alpha.26",
"@octokit/graphql": "^4.5.6",
"@types/express": "^4.17.6",
- "@types/node-fetch": "^2.5.7",
"codeowners-utils": "^1.0.2",
"core-js": "^3.6.5",
+ "cross-fetch": "^3.0.6",
"express": "^4.17.1",
"express-promise-router": "^3.0.3",
"fs-extra": "^9.0.0",
- "git-url-parse": "^11.2.0",
+ "git-url-parse": "^11.4.0",
"knex": "^0.21.1",
"ldapjs": "^2.2.0",
"lodash": "^4.17.15",
"morgan": "^1.10.0",
- "node-fetch": "^2.6.0",
+ "p-limit": "^3.0.2",
"sqlite3": "^5.0.0",
"uuid": "^8.0.0",
"winston": "^3.2.1",
"yaml": "^1.9.2",
"yn": "^4.0.0",
- "yup": "^0.29.1"
+ "yup": "^0.29.3"
},
"devDependencies": {
- "@backstage/cli": "^0.1.1-alpha.24",
+ "@backstage/cli": "^0.1.1-alpha.26",
+ "@backstage/test-utils": "^0.1.1-alpha.26",
"@types/core-js": "^2.5.4",
"@types/git-url-parse": "^9.0.0",
"@types/ldapjs": "^1.0.9",
"@types/lodash": "^4.14.151",
"@types/supertest": "^2.0.8",
"@types/uuid": "^8.0.0",
- "@types/yup": "^0.28.2",
- "jest-fetch-mock": "^3.0.3",
- "msw": "^0.20.5",
+ "@types/yup": "^0.29.8",
+ "msw": "^0.21.2",
"supertest": "^4.0.2"
},
"files": [
diff --git a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts
deleted file mode 100644
index 1cfea23e43..0000000000
--- a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.test.ts
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
-import { Logger } from 'winston';
-import { CoalescedEntitiesCatalog } from './CoalescedEntitiesCatalog';
-import { EntitiesCatalog } from './types';
-
-describe('CoalescedEntitiesCatalog', () => {
- const e1: Entity = {
- apiVersion: 'a',
- kind: 'k',
- metadata: { name: 'n1' },
- };
-
- const e2: Entity = {
- apiVersion: 'a',
- kind: 'k',
- metadata: { name: 'n2' },
- };
-
- const c1: jest.Mocked = {
- entities: jest.fn(),
- entityByUid: jest.fn(),
- entityByName: jest.fn(),
- addOrUpdateEntity: jest.fn(),
- removeEntityByUid: jest.fn(),
- };
-
- const c2: jest.Mocked = {
- entities: jest.fn(),
- entityByUid: jest.fn(),
- entityByName: jest.fn(),
- addOrUpdateEntity: jest.fn(),
- removeEntityByUid: jest.fn(),
- };
-
- const mockLogger = {
- warn: jest.fn(),
- };
- const logger = (mockLogger as unknown) as Logger;
-
- beforeEach(() => {
- jest.resetAllMocks();
- });
-
- describe('entities', () => {
- it('flattens results from multiple sources', async () => {
- c1.entities.mockResolvedValueOnce([e1]);
- c2.entities.mockResolvedValueOnce([e2]);
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(catalog.entities()).resolves.toEqual(
- expect.arrayContaining([e1, e2]),
- );
- expect(c1.entities).toBeCalledTimes(1);
- expect(c2.entities).toBeCalledTimes(1);
- });
-
- it('logs an error if any source throws', async () => {
- c1.entities.mockResolvedValueOnce([e1]);
- c2.entities.mockRejectedValueOnce(new Error('boo'));
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(catalog.entities()).resolves.toEqual([e1]);
- expect(c1.entities).toBeCalledTimes(1);
- expect(c2.entities).toBeCalledTimes(1);
- expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
- });
- });
-
- describe('entityByUid', () => {
- it('returns the first non-undefined result', async () => {
- c1.entityByUid.mockResolvedValueOnce(undefined);
- c2.entityByUid.mockResolvedValueOnce(e2);
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(catalog.entityByUid('e2')).resolves.toBe(e2);
- expect(c1.entityByUid).toBeCalledTimes(1);
- expect(c2.entityByUid).toBeCalledTimes(1);
- });
-
- it('returns undefined if all results were undefined', async () => {
- c1.entityByUid.mockResolvedValueOnce(undefined);
- c2.entityByUid.mockResolvedValueOnce(undefined);
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(catalog.entityByUid('e2')).resolves.toBeUndefined();
- expect(c1.entityByUid).toBeCalledTimes(1);
- expect(c2.entityByUid).toBeCalledTimes(1);
- });
-
- it('logs an error if any source throws', async () => {
- c1.entityByUid.mockResolvedValueOnce(e1);
- c2.entityByUid.mockRejectedValueOnce(new Error('boo'));
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(catalog.entityByUid('e2')).resolves.toBe(e1);
- expect(c1.entityByUid).toBeCalledTimes(1);
- expect(c2.entityByUid).toBeCalledTimes(1);
- expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
- });
- });
-
- describe('entityByName', () => {
- it('returns the first non-undefined result', async () => {
- c1.entityByName.mockResolvedValueOnce(undefined);
- c2.entityByName.mockResolvedValueOnce(e2);
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(
- catalog.entityByName({
- kind: 'k',
- namespace: ENTITY_DEFAULT_NAMESPACE,
- name: 'n2',
- }),
- ).resolves.toBe(e2);
- expect(c1.entityByName).toBeCalledTimes(1);
- expect(c2.entityByName).toBeCalledTimes(1);
- });
-
- it('returns undefined if all results were undefined', async () => {
- c1.entityByName.mockResolvedValueOnce(undefined);
- c2.entityByName.mockResolvedValueOnce(undefined);
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(
- catalog.entityByName({
- kind: 'k',
- namespace: ENTITY_DEFAULT_NAMESPACE,
- name: 'n2',
- }),
- ).resolves.toBeUndefined();
- expect(c1.entityByName).toBeCalledTimes(1);
- expect(c2.entityByName).toBeCalledTimes(1);
- });
-
- it('logs an error if any source throws', async () => {
- c1.entityByName.mockResolvedValueOnce(e1);
- c2.entityByName.mockRejectedValueOnce(new Error('boo'));
- const catalog = new CoalescedEntitiesCatalog([c1, c2], logger);
- await expect(
- catalog.entityByName({
- kind: 'k',
- namespace: ENTITY_DEFAULT_NAMESPACE,
- name: 'n2',
- }),
- ).resolves.toBe(e1);
- expect(c1.entityByName).toBeCalledTimes(1);
- expect(c2.entityByName).toBeCalledTimes(1);
- expect(mockLogger.warn).toBeCalledWith(expect.stringMatching(/boo/));
- });
- });
-});
diff --git a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts
deleted file mode 100644
index 7eaf1868cb..0000000000
--- a/plugins/catalog-backend/src/catalog/CoalescedEntitiesCatalog.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, EntityName } from '@backstage/catalog-model';
-import { Logger } from 'winston';
-import { EntityFilters } from '../database';
-import { EntitiesCatalog } from './types';
-
-/**
- * A simple coalescing catalog wrapper, that acts as a front for collecting
- * catalog data from multiple sources.
- *
- * One possible usage could be to have this as a front to both a
- * DatabaseEntitiesCatalog that holds Component kinds, and another company-
- * specific catalog that is a thin wrapper on top of LDAP that supplies Group
- * and User entities. That way you'll get a coherent view of two very different
- * entity sources.
- *
- * This is mainly meant as a functional example, and you may want to provide
- * your own more specialized collector if you have this distinct need. This
- * one does not support adding/updating entities through the API for example.
- * A more competent implementation may direct the writes to different catalogs
- * based on entity kind or similar.
- */
-export class CoalescedEntitiesCatalog implements EntitiesCatalog {
- private inner: EntitiesCatalog[];
- private logger: Logger;
-
- constructor(inner: EntitiesCatalog[], logger: Logger) {
- this.inner = inner;
- this.logger = logger;
- }
-
- async entities(filters?: EntityFilters): Promise {
- const ops = this.inner.map(async catalog => {
- try {
- return await catalog.entities(filters);
- } catch (e) {
- this.logger.warn(`Inner entities call failed, ${e}`);
- return [];
- }
- });
-
- const results = await Promise.all(ops);
- return results.flat();
- }
-
- async entityByUid(uid: string): Promise {
- const ops = this.inner.map(async catalog => {
- try {
- return await catalog.entityByUid(uid);
- } catch (e) {
- this.logger.warn(`Inner entityByUid call failed, ${e}`);
- return undefined;
- }
- });
-
- const results = await Promise.all(ops);
- return results.find(Boolean);
- }
-
- async entityByName(name: EntityName): Promise {
- const ops = this.inner.map(async catalog => {
- try {
- return await catalog.entityByName(name);
- } catch (e) {
- this.logger.warn(`Inner entityByName call failed, ${e}`);
- return undefined;
- }
- });
-
- const results = await Promise.all(ops);
- return results.find(Boolean);
- }
-
- addOrUpdateEntity(): Promise {
- throw new Error('Method not implemented.');
- }
-
- removeEntityByUid(): Promise {
- throw new Error('Method not implemented.');
- }
-}
diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts
index b94365a9ba..6407000b6f 100644
--- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts
+++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.test.ts
@@ -14,9 +14,11 @@
* limitations under the License.
*/
+import { getVoidLogger } from '@backstage/backend-common';
import type { Entity } from '@backstage/catalog-model';
-import type { Database } from '../database';
+import { Database, DatabaseManager } from '../database';
import { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
+import { EntityUpsertRequest } from './types';
describe('DatabaseEntitiesCatalog', () => {
let db: jest.Mocked;
@@ -24,12 +26,13 @@ describe('DatabaseEntitiesCatalog', () => {
beforeAll(() => {
db = {
transaction: jest.fn(),
- addEntity: jest.fn(),
+ addEntities: jest.fn(),
updateEntity: jest.fn(),
entities: jest.fn(),
entityByName: jest.fn(),
entityByUid: jest.fn(),
- removeEntity: jest.fn(),
+ removeEntityByUid: jest.fn(),
+ setRelations: jest.fn(),
addLocation: jest.fn(),
removeLocation: jest.fn(),
location: jest.fn(),
@@ -44,7 +47,7 @@ describe('DatabaseEntitiesCatalog', () => {
db.transaction.mockImplementation(async f => f('tx'));
});
- describe('addOrUpdateEntity', () => {
+ describe('batchAddOrUpdateEntities', () => {
it('adds when no given uid and no matching by name', async () => {
const entity: Entity = {
apiVersion: 'a',
@@ -56,19 +59,27 @@ describe('DatabaseEntitiesCatalog', () => {
};
db.entities.mockResolvedValue([]);
- db.addEntity.mockResolvedValue({ entity });
+ db.addEntities.mockResolvedValue([
+ { entity: { ...entity, metadata: { ...entity.metadata, uid: 'u' } } },
+ ]);
- const catalog = new DatabaseEntitiesCatalog(db);
- const result = await catalog.addOrUpdateEntity(entity);
+ const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
+ const result = await catalog.batchAddOrUpdateEntities([
+ { entity, relations: [] },
+ ]);
- expect(db.entityByName).toHaveBeenCalledTimes(1);
- expect(db.entityByName).toHaveBeenCalledWith(expect.anything(), {
- kind: 'b',
- namespace: 'd',
- name: 'c',
- });
- expect(db.addEntity).toHaveBeenCalledTimes(1);
- expect(result).toBe(entity);
+ expect(db.entities).toHaveBeenCalledTimes(1);
+ expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+ {
+ kind: 'b',
+ 'metadata.namespace': 'd',
+ 'metadata.name': ['c'],
+ },
+ ]);
+ expect(db.setRelations).toHaveBeenCalledTimes(1);
+ expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
+ expect(db.addEntities).toHaveBeenCalledTimes(1);
+ expect(result).toEqual([{ entityId: 'u' }]);
});
it('updates when given uid', async () => {
@@ -80,9 +91,11 @@ describe('DatabaseEntitiesCatalog', () => {
name: 'c',
namespace: 'd',
},
+ spec: {
+ x: 'b',
+ },
};
-
- db.entityByUid.mockResolvedValue({
+ const existing = {
entity: {
apiVersion: 'a',
kind: 'b',
@@ -93,14 +106,30 @@ describe('DatabaseEntitiesCatalog', () => {
name: 'c',
namespace: 'd',
},
+ spec: {
+ x: 'a',
+ },
},
- });
+ };
+
+ db.entities.mockResolvedValue([existing]);
+ db.entityByUid.mockResolvedValue(existing);
db.updateEntity.mockResolvedValue({ entity });
- const catalog = new DatabaseEntitiesCatalog(db);
- const result = await catalog.addOrUpdateEntity(entity);
+ const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
+ const result = await catalog.batchAddOrUpdateEntities([
+ { entity, relations: [] },
+ ]);
- expect(db.entities).toHaveBeenCalledTimes(0);
+ expect(db.entities).toHaveBeenCalledTimes(1);
+ expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+ {
+ kind: 'b',
+ 'metadata.namespace': 'd',
+ 'metadata.name': ['c'],
+ },
+ ]);
+ expect(db.entityByName).not.toHaveBeenCalled();
expect(db.entityByUid).toHaveBeenCalledTimes(1);
expect(db.entityByUid).toHaveBeenCalledWith(expect.anything(), 'u');
expect(db.updateEntity).toHaveBeenCalledTimes(1);
@@ -112,17 +141,22 @@ describe('DatabaseEntitiesCatalog', () => {
kind: 'b',
metadata: {
uid: 'u',
- etag: 'e',
- generation: 1,
+ etag: expect.any(String),
+ generation: 2,
name: 'c',
namespace: 'd',
},
+ spec: {
+ x: 'b',
+ },
},
},
'e',
1,
);
- expect(result).toBe(entity);
+ expect(db.setRelations).toHaveBeenCalledTimes(1);
+ expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
+ expect(result).toEqual([{ entityId: 'u' }]);
});
it('update when no given uid and matching by name', async () => {
@@ -133,25 +167,44 @@ describe('DatabaseEntitiesCatalog', () => {
name: 'c',
namespace: 'd',
},
+ spec: {
+ x: 'b',
+ },
};
- const existing: Entity = {
- apiVersion: 'a',
- kind: 'b',
- metadata: {
- uid: 'u',
- etag: 'e',
- generation: 1,
- name: 'c',
- namespace: 'd',
+ const existing = {
+ entity: {
+ apiVersion: 'a',
+ kind: 'b',
+ metadata: {
+ uid: 'u',
+ etag: 'e',
+ generation: 1,
+ name: 'c',
+ namespace: 'd',
+ },
+ spec: {
+ x: 'a',
+ },
},
};
- db.entityByName.mockResolvedValue({ entity: existing });
- db.updateEntity.mockResolvedValue({ entity: existing });
+ db.entities.mockResolvedValue([existing]);
+ db.entityByName.mockResolvedValue(existing);
+ db.updateEntity.mockResolvedValue(existing);
- const catalog = new DatabaseEntitiesCatalog(db);
- const result = await catalog.addOrUpdateEntity(added);
+ const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
+ const result = await catalog.batchAddOrUpdateEntities([
+ { entity: added, relations: [] },
+ ]);
+ expect(db.entities).toHaveBeenCalledTimes(1);
+ expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+ {
+ kind: 'b',
+ 'metadata.namespace': 'd',
+ 'metadata.name': ['c'],
+ },
+ ]);
expect(db.entityByName).toHaveBeenCalledTimes(1);
expect(db.entityByName).toHaveBeenCalledWith(expect.anything(), {
kind: 'b',
@@ -167,17 +220,97 @@ describe('DatabaseEntitiesCatalog', () => {
kind: 'b',
metadata: {
uid: 'u',
- etag: 'e',
- generation: 1,
+ etag: expect.any(String),
+ generation: 2,
name: 'c',
namespace: 'd',
},
+ spec: {
+ x: 'b',
+ },
},
},
'e',
1,
);
- expect(result).toEqual(existing);
+ expect(result).toEqual([{ entityId: 'u' }]);
});
+
+ it('should not update if entity is unchanged', async () => {
+ const entity: Entity = {
+ apiVersion: 'a',
+ kind: 'b',
+ metadata: {
+ uid: 'u',
+ name: 'c',
+ namespace: 'd',
+ },
+ spec: {
+ x: 'a',
+ },
+ };
+
+ db.entities.mockResolvedValue([{ entity }]);
+ db.entityByUid.mockResolvedValue({ entity });
+ db.updateEntity.mockResolvedValue({ entity });
+
+ const catalog = new DatabaseEntitiesCatalog(db, getVoidLogger());
+ const result = await catalog.batchAddOrUpdateEntities([
+ { entity, relations: [] },
+ ]);
+
+ expect(db.entities).toHaveBeenCalledTimes(1);
+ expect(db.entities).toHaveBeenCalledWith(expect.anything(), [
+ {
+ kind: 'b',
+ 'metadata.namespace': 'd',
+ 'metadata.name': ['c'],
+ },
+ ]);
+ expect(db.entityByName).not.toHaveBeenCalled();
+ expect(db.entityByUid).not.toHaveBeenCalled();
+ expect(db.updateEntity).not.toHaveBeenCalled();
+ expect(db.setRelations).toHaveBeenCalledTimes(1);
+ expect(db.setRelations).toHaveBeenCalledWith(expect.anything(), 'u', []);
+ expect(result).toEqual([{ entityId: 'u' }]);
+ });
+
+ it('both adds and updates', async () => {
+ const catalog = new DatabaseEntitiesCatalog(
+ await DatabaseManager.createTestDatabase(),
+ getVoidLogger(),
+ );
+ const entities: EntityUpsertRequest[] = [];
+ for (let i = 0; i < 300; ++i) {
+ entities.push({
+ entity: {
+ apiVersion: 'a',
+ kind: 'k',
+ metadata: { name: `n${i}` },
+ },
+ relations: [],
+ });
+ }
+
+ await catalog.batchAddOrUpdateEntities(entities);
+ const afterFirst = await catalog.entities();
+ expect(afterFirst.length).toBe(300);
+
+ entities[40].entity.metadata.op = 'changed';
+ entities.push({
+ entity: {
+ apiVersion: 'a',
+ kind: 'k',
+ metadata: { name: `n300`, op: 'added' },
+ },
+ relations: [],
+ });
+
+ await catalog.batchAddOrUpdateEntities(entities);
+ const afterSecond = await catalog.entities();
+ expect(afterSecond.length).toBe(301);
+ expect(afterSecond.find(e => e.metadata.op === 'changed')).toBeDefined();
+ expect(afterSecond.find(e => e.metadata.op === 'added')).toBeDefined();
+ }, 10000);
});
});
diff --git a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
index 750d728c67..947736dc06 100644
--- a/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+++ b/plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
@@ -14,42 +14,60 @@
* limitations under the License.
*/
-import { NotFoundError } from '@backstage/backend-common';
+import { ConflictError, NotFoundError } from '@backstage/backend-common';
import {
- EntityName,
+ entityHasChanges,
generateUpdatedEntity,
getEntityName,
LOCATION_ANNOTATION,
+ serializeEntityRef,
+ Entity,
+ EntityRelationSpec,
} from '@backstage/catalog-model';
-import type { Entity } from '@backstage/catalog-model';
+import { chunk, groupBy } from 'lodash';
+import limiterFactory from 'p-limit';
+import { Logger } from 'winston';
import type { Database, DbEntityResponse, EntityFilters } from '../database';
-import type { EntitiesCatalog } from './types';
+import { durationText } from '../util/timing';
+import type {
+ EntitiesCatalog,
+ EntityUpsertRequest,
+ EntityUpsertResponse,
+} from './types';
+
+type BatchContext = {
+ kind: string;
+ namespace: string;
+ locationId?: string;
+};
+
+// Some locations return tens or hundreds of thousands of entities. To make
+// those payloads more manageable, we break work apart in batches of this
+// many entities and write them to storage per batch.
+const BATCH_SIZE = 100;
+
+// When writing large batches, there's an increasing chance of contention in
+// the form of conflicts where we compete with other writes. Each batch gets
+// this many attempts at being written before giving up.
+const BATCH_ATTEMPTS = 3;
+
+// The number of batches that may be ongoing at the same time.
+const BATCH_CONCURRENCY = 3;
export class DatabaseEntitiesCatalog implements EntitiesCatalog {
- constructor(private readonly database: Database) {}
+ constructor(
+ private readonly database: Database,
+ private readonly logger: Logger,
+ ) {}
- async entities(filters?: EntityFilters): Promise {
+ async entities(filters?: EntityFilters[]): Promise {
const items = await this.database.transaction(tx =>
this.database.entities(tx, filters),
);
return items.map(i => i.entity);
}
- async entityByUid(uid: string): Promise {
- const response = await this.database.transaction(tx =>
- this.database.entityByUid(tx, uid),
- );
- return response?.entity;
- }
-
- async entityByName(name: EntityName): Promise {
- const response = await this.database.transaction(tx =>
- this.database.entityByName(tx, name),
- );
- return response?.entity;
- }
-
- async addOrUpdateEntity(
+ private async addOrUpdateEntity(
entity: Entity,
locationId?: string,
): Promise {
@@ -72,7 +90,10 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
existing.entity.metadata.generation,
);
} else {
- response = await this.database.addEntity(tx, { locationId, entity });
+ const added = await this.database.addEntities(tx, [
+ { locationId, entity },
+ ]);
+ response = added[0];
}
return response.entity;
@@ -89,14 +110,14 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
entityResponse.entity.metadata.annotations?.[LOCATION_ANNOTATION];
const colocatedEntities = location
? await this.database.entities(tx, [
- {
- key: `metadata.annotations.${LOCATION_ANNOTATION}`,
- values: [location],
- },
+ { [`metadata.annotations.${LOCATION_ANNOTATION}`]: location },
])
: [entityResponse];
for (const dbResponse of colocatedEntities) {
- await this.database.removeEntity(tx, dbResponse?.entity.metadata.uid!);
+ await this.database.removeEntityByUid(
+ tx,
+ dbResponse?.entity.metadata.uid!,
+ );
}
if (entityResponse.locationId) {
@@ -105,4 +126,205 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
return undefined;
});
}
+
+ /**
+ * Writes a number of entities efficiently to storage.
+ *
+ * @param entities Some entities
+ * @param locationId The location that they all belong to
+ */
+ async batchAddOrUpdateEntities(
+ requests: EntityUpsertRequest[],
+ locationId?: string,
+ ): Promise {
+ // Group the entities by unique kind+namespace combinations
+ const entitiesByKindAndNamespace = groupBy(requests, ({ entity }) => {
+ const name = getEntityName(entity);
+ return `${name.kind}:${name.namespace}`.toLowerCase();
+ });
+
+ const limiter = limiterFactory(BATCH_CONCURRENCY);
+ const tasks: Promise[] = [];
+
+ for (const groupRequests of Object.values(entitiesByKindAndNamespace)) {
+ const { kind, namespace } = getEntityName(groupRequests[0].entity);
+
+ // Go through the new entities in reasonable chunk sizes (sometimes,
+ // sources produce tens of thousands of entities, and those are too large
+ // batch sizes to reasonably send to the database)
+ for (const batch of chunk(groupRequests, BATCH_SIZE)) {
+ tasks.push(
+ limiter(async () => {
+ const first = serializeEntityRef(batch[0].entity);
+ const last = serializeEntityRef(batch[batch.length - 1].entity);
+ const modifiedEntityIds: EntityUpsertResponse[] = [];
+ this.logger.debug(
+ `Considering batch ${first}-${last} (${batch.length} entries)`,
+ );
+
+ // Retry the batch write a few times to deal with contention
+ const context = { kind, namespace, locationId };
+ for (let attempt = 1; attempt <= BATCH_ATTEMPTS; ++attempt) {
+ try {
+ const { toAdd, toUpdate, toIgnore } = await this.analyzeBatch(
+ batch,
+ context,
+ );
+ if (toAdd.length) {
+ modifiedEntityIds.push(
+ ...(await this.batchAdd(toAdd, context)),
+ );
+ }
+ if (toUpdate.length) {
+ modifiedEntityIds.push(
+ ...(await this.batchUpdate(toUpdate, context)),
+ );
+ }
+ // TODO(Rugvip): We currently always update relations, but we
+ // likely want to figure out a way to avoid that
+ for (const { entity, relations } of toIgnore) {
+ const entityId = entity.metadata.uid!;
+ await this.setRelations(entityId, relations);
+ modifiedEntityIds.push({ entityId });
+ }
+
+ break;
+ } catch (e) {
+ if (e instanceof ConflictError && attempt < BATCH_ATTEMPTS) {
+ this.logger.warn(
+ `Failed to write batch at attempt ${attempt}/${BATCH_ATTEMPTS}, ${e}`,
+ );
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ return modifiedEntityIds;
+ }),
+ );
+ }
+ }
+
+ return (await Promise.all(tasks)).flat();
+ }
+
+ // Set the relations originating from an entity using the DB layer
+ private async setRelations(
+ originatingEntityId: string,
+ relations: EntityRelationSpec[],
+ ): Promise {
+ return await this.database.transaction(tx =>
+ this.database.setRelations(tx, originatingEntityId, relations),
+ );
+ }
+
+ // Given a batch of entities that were just read from a location, take them
+ // into consideration by comparing against the existing catalog entities and
+ // produce the list of entities to be added, and the list of entities to be
+ // updated
+ private async analyzeBatch(
+ requests: EntityUpsertRequest[],
+ { kind, namespace }: BatchContext,
+ ): Promise<{
+ toAdd: EntityUpsertRequest[];
+ toUpdate: EntityUpsertRequest[];
+ toIgnore: EntityUpsertRequest[];
+ }> {
+ const markTimestamp = process.hrtime();
+
+ const names = requests.map(({ entity }) => entity.metadata.name);
+ const oldEntities = await this.entities([
+ {
+ kind: kind,
+ 'metadata.namespace': namespace,
+ 'metadata.name': names,
+ },
+ ]);
+
+ const oldEntitiesByName = new Map(
+ oldEntities.map(e => [e.metadata.name, e]),
+ );
+
+ const toAdd: EntityUpsertRequest[] = [];
+ const toUpdate: EntityUpsertRequest[] = [];
+ const toIgnore: EntityUpsertRequest[] = [];
+
+ for (const request of requests) {
+ const newEntity = request.entity;
+ const oldEntity = oldEntitiesByName.get(newEntity.metadata.name);
+ if (!oldEntity) {
+ toAdd.push(request);
+ } else if (entityHasChanges(oldEntity, newEntity)) {
+ // TODO(freben): This currently uses addOrUpdateEntity under the hood,
+ // but should probably calculate the end result entity right here
+ // instead and call a dedicated batch update database method instead
+ toUpdate.push(request);
+ } else {
+ toIgnore.push(request);
+ }
+ }
+
+ this.logger.debug(
+ `Found ${toAdd.length} entities to add, ${
+ toUpdate.length
+ } entities to update in ${durationText(markTimestamp)}`,
+ );
+
+ return { toAdd, toUpdate, toIgnore };
+ }
+
+ // Efficiently adds the given entities to storage, under the assumption that
+ // they do not conflict with any existing entities
+ private async batchAdd(
+ requests: EntityUpsertRequest[],
+ { locationId }: BatchContext,
+ ): Promise {
+ const markTimestamp = process.hrtime();
+
+ const res = await this.database.transaction(
+ async tx =>
+ await this.database.addEntities(
+ tx,
+ requests.map(({ entity }) => ({ locationId, entity })),
+ ),
+ );
+
+ const entityIds = res.map(({ entity }) => ({
+ entityId: entity.metadata.uid!,
+ }));
+
+ for (const [index, { entityId }] of entityIds.entries()) {
+ await this.setRelations(entityId, requests[index].relations);
+ }
+
+ this.logger.debug(
+ `Added ${requests.length} entities in ${durationText(markTimestamp)}`,
+ );
+
+ return entityIds;
+ }
+
+ // Efficiently updates the given entities into storage, under the assumption
+ // that there already exist entities with the same names
+ private async batchUpdate(
+ requests: EntityUpsertRequest[],
+ { locationId }: BatchContext,
+ ): Promise {
+ const markTimestamp = process.hrtime();
+ const responseIds: EntityUpsertResponse[] = [];
+ // TODO(freben): Still not batched
+ for (const entity of requests) {
+ const res = await this.addOrUpdateEntity(entity.entity, locationId);
+ const entityId = res.metadata.uid!;
+ responseIds.push({ entityId });
+ await this.setRelations(entityId, entity.relations);
+ }
+
+ this.logger.debug(
+ `Updated ${requests.length} entities in ${durationText(markTimestamp)}`,
+ );
+
+ return responseIds;
+ }
}
diff --git a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts b/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
deleted file mode 100644
index 65fbfb9071..0000000000
--- a/plugins/catalog-backend/src/catalog/StaticEntitiesCatalog.ts
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { Entity, EntityName, getEntityName } from '@backstage/catalog-model';
-import lodash from 'lodash';
-import type { EntitiesCatalog } from './types';
-
-export class StaticEntitiesCatalog implements EntitiesCatalog {
- private _entities: Entity[];
-
- constructor(entities: Entity[]) {
- this._entities = entities;
- }
-
- async entities(): Promise {
- return lodash.cloneDeep(this._entities);
- }
-
- async entityByUid(uid: string): Promise {
- const item = this._entities.find(e => uid === e.metadata.uid);
- return item ? lodash.cloneDeep(item) : undefined;
- }
-
- async entityByName(name: EntityName): Promise {
- const item = this._entities.find(e => {
- const candidate = getEntityName(e);
- return (
- name.kind.toLowerCase() === candidate.kind.toLowerCase() &&
- name.namespace.toLowerCase() === candidate.namespace.toLowerCase() &&
- name.name.toLowerCase() === candidate.name.toLowerCase()
- );
- });
- return item ? lodash.cloneDeep(item) : undefined;
- }
-
- async addOrUpdateEntity(): Promise {
- throw new Error('Not supported');
- }
-
- async removeEntityByUid(): Promise {
- throw new Error('Not supported');
- }
-}
diff --git a/plugins/catalog-backend/src/catalog/index.ts b/plugins/catalog-backend/src/catalog/index.ts
index 308078b1fc..ce429327a0 100644
--- a/plugins/catalog-backend/src/catalog/index.ts
+++ b/plugins/catalog-backend/src/catalog/index.ts
@@ -16,5 +16,4 @@
export { DatabaseEntitiesCatalog } from './DatabaseEntitiesCatalog';
export { DatabaseLocationsCatalog } from './DatabaseLocationsCatalog';
-export { StaticEntitiesCatalog } from './StaticEntitiesCatalog';
export type { EntitiesCatalog, LocationsCatalog } from './types';
diff --git a/plugins/catalog-backend/src/catalog/types.ts b/plugins/catalog-backend/src/catalog/types.ts
index 37618a2440..539a0e531f 100644
--- a/plugins/catalog-backend/src/catalog/types.ts
+++ b/plugins/catalog-backend/src/catalog/types.ts
@@ -14,19 +14,36 @@
* limitations under the License.
*/
-import { Entity, EntityName, Location } from '@backstage/catalog-model';
+import { Entity, EntityRelationSpec, Location } from '@backstage/catalog-model';
import type { EntityFilters } from '../database';
//
// Entities
//
+export type EntityUpsertRequest = {
+ entity: Entity;
+ relations: EntityRelationSpec[];
+};
+
+export type EntityUpsertResponse = {
+ entityId: string;
+};
+
export type EntitiesCatalog = {
- entities(filters?: EntityFilters): Promise;
- entityByUid(uid: string): Promise;
- entityByName(name: EntityName): Promise;
- addOrUpdateEntity(entity: Entity, locationId?: string): Promise;
+ entities(filters?: EntityFilters[]): Promise;
removeEntityByUid(uid: string): Promise;
+
+ /**
+ * Writes a number of entities efficiently to storage.
+ *
+ * @param entities Some entities
+ * @param locationId The location that they all belong to
+ */
+ batchAddOrUpdateEntities(
+ entities: EntityUpsertRequest[],
+ locationId?: string,
+ ): Promise;
};
//
diff --git a/plugins/catalog-backend/src/database/CommonDatabase.test.ts b/plugins/catalog-backend/src/database/CommonDatabase.test.ts
index 8a42b39cc5..ac11f1dd1a 100644
--- a/plugins/catalog-backend/src/database/CommonDatabase.test.ts
+++ b/plugins/catalog-backend/src/database/CommonDatabase.test.ts
@@ -15,14 +15,14 @@
*/
import { ConflictError } from '@backstage/backend-common';
-import type { Entity, Location } from '@backstage/catalog-model';
+import { Entity, Location, parseEntityRef } from '@backstage/catalog-model';
import { DatabaseManager } from './DatabaseManager';
-import { Database, DatabaseLocationUpdateLogStatus } from './types';
import type {
DbEntityRequest,
DbEntityResponse,
DbLocationsRowWithStatus,
} from './types';
+import { Database, DatabaseLocationUpdateLogStatus } from './types';
const bootstrapLocation = {
id: expect.any(String),
@@ -124,7 +124,7 @@ describe('CommonDatabase', () => {
{
...output,
status: DatabaseLocationUpdateLogStatus.FAIL,
- timestamp: expect.any(String),
+ timestamp: expect.anything(),
},
]),
);
@@ -135,36 +135,99 @@ describe('CommonDatabase', () => {
await expect(db.location(location.id)).rejects.toThrow(/Found no location/);
});
- describe('addEntity', () => {
- it('happy path: adds entity to empty database', async () => {
- const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
- expect(added).toStrictEqual(entityResponse);
- expect(added.entity.metadata.generation).toBe(1);
+ describe('addEntities', () => {
+ it('happy path: adds entities to empty database', async () => {
+ const result = await db.transaction(tx =>
+ db.addEntities(tx, [entityRequest]),
+ );
+ expect(result).toEqual([entityResponse]);
});
it('rejects adding the same-named entity twice', async () => {
- await db.transaction(tx => db.addEntity(tx, entityRequest));
+ const req: DbEntityRequest[] = [
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'ns1' },
+ },
+ },
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'ns1' },
+ },
+ },
+ ];
await expect(
- db.transaction(tx => db.addEntity(tx, entityRequest)),
+ db.transaction(tx => db.addEntities(tx, req)),
).rejects.toThrow(ConflictError);
});
it('rejects adding the almost-same-namespace entity twice', async () => {
- entityRequest.entity.metadata.namespace = undefined;
- await db.transaction(tx => db.addEntity(tx, entityRequest));
- entityRequest.entity.metadata.namespace = '';
+ const req: DbEntityRequest[] = [
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'ns1' },
+ },
+ },
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'nS1' },
+ },
+ },
+ ];
await expect(
- db.transaction(tx => db.addEntity(tx, entityRequest)),
+ db.transaction(tx => db.addEntities(tx, req)),
).rejects.toThrow(ConflictError);
});
it('accepts adding the same-named entity twice if on different namespaces', async () => {
- entityRequest.entity.metadata.namespace = 'namespace1';
- await db.transaction(tx => db.addEntity(tx, entityRequest));
- entityRequest.entity.metadata.namespace = 'namespace2';
+ const req: DbEntityRequest[] = [
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'ns1' },
+ },
+ },
+ {
+ entity: {
+ apiVersion: 'av1',
+ kind: 'k1',
+ metadata: { name: 'n1', namespace: 'ns2' },
+ },
+ },
+ ];
await expect(
- db.transaction(tx => db.addEntity(tx, entityRequest)),
- ).resolves.toBeDefined();
+ db.transaction(tx => db.addEntities(tx, req)),
+ ).resolves.toEqual([
+ {
+ entity: expect.objectContaining({
+ metadata: expect.objectContaining({
+ namespace: 'ns1',
+ uid: expect.any(String),
+ etag: expect.any(String),
+ generation: expect.any(Number),
+ }),
+ }),
+ },
+ {
+ entity: expect.objectContaining({
+ metadata: expect.objectContaining({
+ namespace: 'ns2',
+ uid: expect.any(String),
+ etag: expect.any(String),
+ generation: expect.any(Number),
+ }),
+ }),
+ },
+ ]);
});
});
@@ -191,30 +254,34 @@ describe('CommonDatabase', () => {
const result = await db.locationHistory(
'dd12620d-0436-422f-93bd-929aa0788123',
);
- expect(result).toEqual([
- {
- created_at: expect.anything(),
- entity_name: null,
- id: expect.anything(),
- location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
- message: null,
- status: DatabaseLocationUpdateLogStatus.SUCCESS,
- },
- {
- created_at: expect.anything(),
- entity_name: null,
- id: expect.anything(),
- location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
- message: 'Something went wrong',
- status: DatabaseLocationUpdateLogStatus.FAIL,
- },
- ]);
+ expect(result).toEqual(
+ expect.arrayContaining([
+ {
+ created_at: expect.anything(),
+ entity_name: null,
+ id: expect.anything(),
+ location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
+ message: null,
+ status: DatabaseLocationUpdateLogStatus.SUCCESS,
+ },
+ {
+ created_at: expect.anything(),
+ entity_name: null,
+ id: expect.anything(),
+ location_id: 'dd12620d-0436-422f-93bd-929aa0788123',
+ message: 'Something went wrong',
+ status: DatabaseLocationUpdateLogStatus.FAIL,
+ },
+ ]),
+ );
});
});
describe('updateEntity', () => {
it('can read and no-op-update an entity', async () => {
- const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+ const [added] = await db.transaction(tx =>
+ db.addEntities(tx, [entityRequest]),
+ );
const updated = await db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity }),
);
@@ -231,7 +298,9 @@ describe('CommonDatabase', () => {
});
it('can update name if uid matches', async () => {
- const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+ const [added] = await db.transaction(tx =>
+ db.addEntities(tx, [entityRequest]),
+ );
added.entity.metadata.name! = 'new!';
const updated = await db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity }),
@@ -240,7 +309,9 @@ describe('CommonDatabase', () => {
});
it('fails to update an entity if etag does not match', async () => {
- const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+ const [added] = await db.transaction(tx =>
+ db.addEntities(tx, [entityRequest]),
+ );
await expect(
db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity }, 'garbage'),
@@ -249,7 +320,9 @@ describe('CommonDatabase', () => {
});
it('fails to update an entity if generation does not match', async () => {
- const added = await db.transaction(tx => db.addEntity(tx, entityRequest));
+ const [added] = await db.transaction(tx =>
+ db.addEntities(tx, [entityRequest]),
+ );
await expect(
db.transaction(tx =>
db.updateEntity(tx, { entity: added.entity }, undefined, 1e20),
@@ -272,8 +345,7 @@ describe('CommonDatabase', () => {
spec: { c: null },
};
await db.transaction(async tx => {
- await db.addEntity(tx, { entity: e1 });
- await db.addEntity(tx, { entity: e2 });
+ await db.addEntities(tx, [{ entity: e1 }, { entity: e2 }]);
});
const result = await db.transaction(async tx => db.entities(tx, []));
expect(result.length).toEqual(2);
@@ -309,17 +381,15 @@ describe('CommonDatabase', () => {
];
await db.transaction(async tx => {
- for (const entity of entities) {
- await db.addEntity(tx, { entity });
- }
+ await db.addEntities(
+ tx,
+ entities.map(entity => ({ entity })),
+ );
});
await expect(
db.transaction(async tx =>
- db.entities(tx, [
- { key: 'kind', values: ['k2'] },
- { key: 'spec.c', values: ['some'] },
- ]),
+ db.entities(tx, [{ kind: 'k2', 'spec.c': 'some' }]),
),
).resolves.toEqual([
{
@@ -347,16 +417,14 @@ describe('CommonDatabase', () => {
];
await db.transaction(async tx => {
- for (const entity of entities) {
- await db.addEntity(tx, { entity });
- }
+ await db.addEntities(
+ tx,
+ entities.map(entity => ({ entity })),
+ );
});
const rows = await db.transaction(async tx =>
- db.entities(tx, [
- { key: 'apiVersion', values: ['a'] },
- { key: 'spec.c', values: [null, 'some'] },
- ]),
+ db.entities(tx, [{ apiVersion: 'a', 'spec.c': [null, 'some'] }]),
);
expect(rows.length).toEqual(3);
@@ -396,16 +464,14 @@ describe('CommonDatabase', () => {
];
await db.transaction(async tx => {
- for (const entity of entities) {
- await db.addEntity(tx, { entity });
- }
+ await db.addEntities(
+ tx,
+ entities.map(entity => ({ entity })),
+ );
});
const rows = await db.transaction(async tx =>
- db.entities(tx, [
- { key: 'ApiVersioN', values: ['A'] },
- { key: 'spEc.C', values: [null, 'some'] },
- ]),
+ db.entities(tx, [{ ApiVersioN: 'A', 'spEc.C': [null, 'some'] }]),
);
expect(rows.length).toEqual(3);
@@ -428,6 +494,196 @@ describe('CommonDatabase', () => {
});
});
+ describe('setRelations', () => {
+ it('adds a relation for an entity', async () => {
+ const mockRelations = [
+ {
+ source: {
+ kind: entityRequest.entity.kind,
+ namespace: entityRequest.entity.metadata.namespace!,
+ name: entityRequest.entity.metadata.name,
+ },
+ target: {
+ kind: 'component',
+ namespace: 'asd',
+ name: 'bleb',
+ },
+ type: 'child',
+ },
+ ];
+
+ const entityId = await db.transaction(async tx => {
+ const [{ entity }] = await db.addEntities(tx, [entityRequest]);
+
+ await db.setRelations(tx, entity?.metadata?.uid!, mockRelations);
+ return entity.metadata.uid;
+ });
+
+ const returnedEntity1 = await db.transaction(tx =>
+ db.entityByUid(tx, entityId!),
+ );
+ expect(returnedEntity1?.entity.relations).toEqual([
+ { target: mockRelations[0].target, type: 'child' },
+ ]);
+
+ const returnedEntity2 = await db.transaction(tx =>
+ db.entityByName(tx, mockRelations[0].source),
+ );
+ expect(returnedEntity2?.entity.relations).toEqual([
+ { target: mockRelations[0].target, type: 'child' },
+ ]);
+
+ const [returnedEntity3] = await db.transaction(tx => db.entities(tx));
+ expect(returnedEntity3?.entity.relations).toEqual([
+ { target: mockRelations[0].target, type: 'child' },
+ ]);
+ });
+
+ function makeRelation(source: string, type: string, target: string) {
+ return {
+ source: parseEntityRef(source, {
+ defaultKind: 'x',
+ defaultNamespace: 'x',
+ }),
+ type,
+ target: parseEntityRef(target, {
+ defaultKind: 'x',
+ defaultNamespace: 'x',
+ }),
+ };
+ }
+
+ it('should not allow setting relations on nonexistent entities', async () => {
+ await expect(
+ db.transaction(async tx => {
+ await db.setRelations(tx, 'nonexistent', [
+ makeRelation('a:b/c', 'rel1', 'x:y/z'),
+ ]);
+ }),
+ ).rejects.toThrow(/constraint failed/);
+ });
+
+ it('should allow setting relations on nonexistent entities without any relations', async () => {
+ await expect(
+ db.transaction(async tx => {
+ await db.setRelations(tx, 'nonexistent', []);
+ }),
+ ).resolves.toBeUndefined();
+ });
+
+ it('adds multiple relations for entities', async () => {
+ const entity1 = {
+ apiVersion: 'v1',
+ kind: 'a',
+ metadata: {
+ name: 'c',
+ namespace: 'b',
+ },
+ };
+ const entity2 = {
+ apiVersion: 'v1',
+ kind: 'x',
+ metadata: {
+ name: 'z',
+ namespace: 'y',
+ },
+ };
+ const fromEntity1 = [
+ makeRelation('a:b/c', 'rel1', 'x:y/z'),
+ makeRelation('x:y/z', 'rel2', 'a:b/c'),
+ makeRelation('a:b/c', 'rel2', 'x:y/z'),
+ ];
+ const fromEntity2 = [
+ makeRelation('a:b/c', 'rel4', 'x:y/z'),
+ makeRelation('a:b/c', 'rel5', 'x:y/z'),
+ makeRelation('x:y/z', 'rel6', 'a:b/c'),
+ // relations don't have to reference the originating entity, so this should be fine, but not show up
+ makeRelation('g:h/i', 'rel8', 'd:e/f'),
+ ];
+
+ const { id2: secondEntityId } = await db.transaction(async tx => {
+ const [{ entity: e1 }, { entity: e2 }] = await db.addEntities(tx, [
+ { entity: entity1 },
+ { entity: entity2 },
+ ]);
+ const id1 = e1?.metadata?.uid!;
+ const id2 = e2?.metadata?.uid!;
+
+ await db.setRelations(tx, id1, fromEntity1);
+ await db.setRelations(tx, id2, fromEntity2);
+
+ return { id1, id2 };
+ });
+
+ const res = await db.transaction(tx => db.entities(tx));
+ expect(
+ res.map(r => ({
+ name: r.entity.metadata.name,
+ relations: r.entity.relations,
+ })),
+ ).toEqual([
+ {
+ name: 'c',
+ relations: [
+ {
+ type: 'rel1',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ {
+ type: 'rel2',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ {
+ type: 'rel4',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ {
+ type: 'rel5',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ ],
+ },
+ {
+ name: 'z',
+ relations: [
+ {
+ type: 'rel2',
+ target: { kind: 'a', namespace: 'b', name: 'c' },
+ },
+ {
+ type: 'rel6',
+ target: { kind: 'a', namespace: 'b', name: 'c' },
+ },
+ ],
+ },
+ ]);
+
+ await db.transaction(tx => db.removeEntityByUid(tx, secondEntityId));
+
+ const res2 = await db.transaction(tx => db.entities(tx));
+ expect(
+ res2.map(r => ({
+ name: r.entity.metadata.name,
+ relations: r.entity.relations,
+ })),
+ ).toEqual([
+ {
+ name: 'c',
+ relations: [
+ {
+ type: 'rel1',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ {
+ type: 'rel2',
+ target: { kind: 'x', namespace: 'y', name: 'z' },
+ },
+ ],
+ },
+ ]);
+ });
+ });
+
describe('entityByName', () => {
it('can get entities case insensitively', async () => {
const entities: Entity[] = [
@@ -444,9 +700,10 @@ describe('CommonDatabase', () => {
];
await db.transaction(async tx => {
- for (const entity of entities) {
- await db.addEntity(tx, { entity });
- }
+ await db.addEntities(
+ tx,
+ entities.map(entity => ({ entity })),
+ );
});
const e1 = await db.transaction(async tx =>
diff --git a/plugins/catalog-backend/src/database/CommonDatabase.ts b/plugins/catalog-backend/src/database/CommonDatabase.ts
index 5327384593..72e6bf0a5f 100644
--- a/plugins/catalog-backend/src/database/CommonDatabase.ts
+++ b/plugins/catalog-backend/src/database/CommonDatabase.ts
@@ -21,22 +21,24 @@ import {
} from '@backstage/backend-common';
import {
Entity,
- EntityMeta,
EntityName,
+ EntityRelationSpec,
ENTITY_DEFAULT_NAMESPACE,
ENTITY_META_GENERATED_FIELDS,
generateEntityEtag,
generateEntityUid,
Location,
+ parseEntityName,
} from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import type { Logger } from 'winston';
import { buildEntitySearch } from './search';
-import type {
+import {
Database,
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
+ DbEntitiesRelationsRow,
DbEntitiesRow,
DbEntitiesSearchRow,
DbEntityRequest,
@@ -46,6 +48,12 @@ import type {
EntityFilters,
} from './types';
+// The number of items that are sent per batch to the database layer, when
+// doing .batchInsert calls to knex. This needs to be low enough to not cause
+// errors in the underlying engine due to exceeding query limits, but large
+// enough to get the speed benefits.
+const BATCH_SIZE = 50;
+
/**
* The core database implementation.
*/
@@ -101,6 +109,54 @@ export class CommonDatabase implements Database {
return { locationId: request.locationId, entity: newEntity };
}
+ async addEntities(
+ txOpaque: unknown,
+ request: DbEntityRequest[],
+ ): Promise {
+ const tx = txOpaque as Knex.Transaction;
+
+ const result: DbEntityResponse[] = [];
+ const entityRows: DbEntitiesRow[] = [];
+ const searchRows: DbEntitiesSearchRow[] = [];
+
+ for (const { entity, locationId } of request) {
+ if (entity.metadata.uid !== undefined) {
+ throw new InputError('May not specify uid for new entities');
+ } else if (entity.metadata.etag !== undefined) {
+ throw new InputError('May not specify etag for new entities');
+ } else if (entity.metadata.generation !== undefined) {
+ throw new InputError('May not specify generation for new entities');
+ } else if (entity.relations !== undefined) {
+ throw new InputError('May not specify relations for new entities');
+ }
+
+ const newEntity = {
+ ...entity,
+ metadata: {
+ ...entity.metadata,
+ uid: generateEntityUid(),
+ etag: generateEntityEtag(),
+ generation: 1,
+ },
+ };
+
+ result.push({ entity: newEntity, locationId });
+ entityRows.push(this.toEntityRow(locationId, newEntity));
+ searchRows.push(...buildEntitySearch(newEntity.metadata.uid, newEntity));
+ }
+
+ await tx.batchInsert('entities', entityRows, BATCH_SIZE);
+ await tx('entities_search')
+ .whereIn(
+ 'entity_id',
+ entityRows.map(r => r.id),
+ )
+ .del();
+ await tx.batchInsert('entities_search', searchRows, BATCH_SIZE);
+
+ return result;
+ }
+
async updateEntity(
txOpaque: unknown,
request: DbEntityRequest,
@@ -162,58 +218,80 @@ export class CommonDatabase implements Database {
async entities(
txOpaque: unknown,
- filters?: EntityFilters,
+ filters?: EntityFilters[],
): Promise {
const tx = txOpaque as Knex.Transaction;
- let builder = tx('entities');
- for (const [indexU, filter] of (filters ?? []).entries()) {
- const index = Number(indexU);
- const key = filter.key.toLowerCase().replace(/\*/g, '%');
- const keyOp = filter.key.includes('*') ? 'like' : '=';
+ let entitiesQuery = tx('entities');
- let matchNulls = false;
- const matchIn: string[] = [];
- const matchLike: string[] = [];
+ for (const singleFilter of filters ?? []) {
+ entitiesQuery = entitiesQuery.orWhere(function singleFilterFn() {
+ for (const [matchKey, matchVal] of Object.entries(singleFilter)) {
+ const key = matchKey.toLowerCase().replace(/[*]/g, '%');
+ const keyOp = key.includes('%') ? 'like' : '=';
+ const values = Array.isArray(matchVal) ? matchVal : [matchVal];
- for (const value of filter.values) {
- if (!value) {
- matchNulls = true;
- } else if (value.includes('*')) {
- matchLike.push(value.toLowerCase().replace(/\*/g, '%'));
- } else {
- matchIn.push(value.toLowerCase());
- }
- }
+ let matchNulls = false;
+ const matchIn: string[] = [];
+ const matchLike: string[] = [];
- builder = builder
- .leftOuterJoin(`entities_search as t${index}`, function joins() {
- this.on('entities.id', '=', `t${index}.entity_id`);
- this.andOn(`t${index}.key`, keyOp, tx.raw('?', [key]));
- })
- .where(function rules() {
- if (matchIn.length) {
- this.orWhereIn(`t${index}.value`, matchIn);
- }
- if (matchLike.length) {
- for (const x of matchLike) {
- this.orWhere(`t${index}.value`, 'like', tx.raw('?', [x]));
+ for (const value of values) {
+ if (!value) {
+ matchNulls = true;
+ } else if (value.includes('*')) {
+ matchLike.push(value.toLowerCase().replace(/[*]/g, '%'));
+ } else {
+ matchIn.push(value.toLowerCase());
}
}
- if (matchNulls) {
- this.orWhereNull(`t${index}.value`);
- }
- });
+
+ // NOTE(freben): This used to be a set of OUTER JOIN, which may seem to
+ // make a lot of sense. However, it had abysmal performance on sqlite
+ // when datasets grew large, so we're using IN instead.
+ const matchQuery = tx('entities_search')
+ .select('entity_id')
+ .where(function keyFilter() {
+ this.andWhere('key', keyOp, key);
+ this.andWhere(function valueFilter() {
+ if (matchIn.length === 1) {
+ this.orWhere({ value: matchIn[0] });
+ } else if (matchIn.length > 1) {
+ this.orWhereIn('value', matchIn);
+ }
+ if (matchLike.length) {
+ for (const x of matchLike) {
+ this.orWhere('value', 'like', tx.raw('?', [x]));
+ }
+ }
+ if (matchNulls) {
+ // Match explicit nulls, and then handle absence separately
+ // below
+ this.orWhereNull('value');
+ }
+ });
+ });
+
+ // Handle absence as nulls as well
+ this.andWhere(function match() {
+ this.whereIn('id', matchQuery);
+ if (matchNulls) {
+ this.orWhereNotIn(
+ 'id',
+ tx('entities_search')
+ .select('entity_id')
+ .where('key', keyOp, key),
+ );
+ }
+ });
+ }
+ });
}
- const rows = await builder
+ const rows = await entitiesQuery
.select('entities.*')
- .orderBy('kind', 'asc')
- .orderBy('namespace', 'asc')
- .orderBy('name', 'asc')
- .groupBy('id');
+ .orderBy('full_name', 'asc');
- return rows.map(row => this.toEntityResponse(row));
+ return Promise.all(rows.map(row => this.toEntityResponse(tx, row)));
}
async entityByName(
@@ -223,19 +301,16 @@ export class CommonDatabase implements Database {
const tx = txOpaque as Knex.Transaction;
const rows = await tx('entities')
- .whereRaw(
- tx.raw(
- 'LOWER(kind) = LOWER(?) AND LOWER(namespace) = LOWER(?) AND LOWER(name) = LOWER(?)',
- [name.kind, name.namespace, name.name],
- ),
- )
+ .where({
+ full_name: `${name.kind}:${name.namespace}/${name.name}`.toLowerCase(),
+ })
.select();
if (rows.length !== 1) {
return undefined;
}
- return this.toEntityResponse(rows[0]);
+ return this.toEntityResponse(tx, rows[0]);
}
async entityByUid(
@@ -252,10 +327,10 @@ export class CommonDatabase implements Database {
return undefined;
}
- return this.toEntityResponse(rows[0]);
+ return this.toEntityResponse(tx, rows[0]);
}
- async removeEntity(txOpaque: unknown, uid: string): Promise {
+ async removeEntityByUid(txOpaque: unknown, uid: string): Promise {
const tx = txOpaque as Knex.Transaction;
const result = await tx('entities').where({ id: uid }).del();
@@ -265,6 +340,34 @@ export class CommonDatabase implements Database {
}
}
+ async setRelations(
+ txOpaque: unknown,
+ originatingEntityId: string,
+ relations: EntityRelationSpec[],
+ ): Promise {
+ const tx = txOpaque as Knex.Transaction;
+
+ // remove all relations that exist for the originating entity id.
+ await tx('entities_relations')
+ .where({ originating_entity_id: originatingEntityId })
+ .del();
+
+ const serializeName = (e: EntityName) =>
+ `${e.kind}:${e.namespace}/${e.name}`.toLowerCase();
+
+ const relationsRows: DbEntitiesRelationsRow[] = relations.map(
+ ({ source, target, type }) => ({
+ originating_entity_id: originatingEntityId,
+ source_full_name: serializeName(source),
+ target_full_name: serializeName(target),
+ type,
+ }),
+ );
+
+ // TODO(blam): translate constraint failures to sane NotFoundError instead
+ await tx.batchInsert('entities_relations', relationsRows, BATCH_SIZE);
+ }
+
async addLocation(location: Location): Promise {
return await this.database.transaction(async tx => {
const row: DbLocationsRow = {
@@ -389,39 +492,41 @@ export class CommonDatabase implements Database {
).toLowerCase();
const lowerName = entity.metadata.name.toLowerCase();
+ const data = {
+ ...entity,
+ metadata: lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
+ };
+
return {
id: entity.metadata.uid!,
location_id: locationId || null,
etag: entity.metadata.etag!,
generation: entity.metadata.generation!,
full_name: `${lowerKind}:${lowerNamespace}/${lowerName}`,
- api_version: entity.apiVersion,
- kind: entity.kind,
- name: entity.metadata.name,
- namespace: entity.metadata.namespace || ENTITY_DEFAULT_NAMESPACE,
- metadata: JSON.stringify(
- lodash.omit(entity.metadata, ...ENTITY_META_GENERATED_FIELDS),
- ),
- spec: entity.spec ? JSON.stringify(entity.spec) : null,
+ data: JSON.stringify(data),
};
}
- private toEntityResponse(row: DbEntitiesRow): DbEntityResponse {
- const entity: Entity = {
- apiVersion: row.api_version,
- kind: row.kind,
- metadata: {
- ...(JSON.parse(row.metadata) as EntityMeta),
- uid: row.id,
- etag: row.etag,
- generation: Number(row.generation), // cast because of sqlite
- },
- };
+ private async toEntityResponse(
+ tx: Knex.Transaction,
+ row: DbEntitiesRow,
+ ): Promise {
+ const entity = JSON.parse(row.data) as Entity;
+ entity.metadata.uid = row.id;
+ entity.metadata.etag = row.etag;
+ entity.metadata.generation = Number(row.generation); // cast due to sqlite
- if (row.spec) {
- const spec = JSON.parse(row.spec);
- entity.spec = spec;
- }
+ // TODO(Rugvip): This is here because it's simple for now, but we likely
+ // need to refactor this to be more efficient or introduce pagination.
+ const relations = await tx('entities_relations')
+ .where({ source_full_name: row.full_name })
+ .orderBy(['type', 'target_full_name'])
+ .select();
+
+ entity.relations = relations.map(r => ({
+ target: parseEntityName(r.target_full_name),
+ type: r.type,
+ }));
return {
locationId: row.location_id || undefined,
diff --git a/plugins/catalog-backend/src/database/DatabaseManager.ts b/plugins/catalog-backend/src/database/DatabaseManager.ts
index afff254097..465bf654f2 100644
--- a/plugins/catalog-backend/src/database/DatabaseManager.ts
+++ b/plugins/catalog-backend/src/database/DatabaseManager.ts
@@ -19,6 +19,7 @@ import Knex from 'knex';
import { Logger } from 'winston';
import { CommonDatabase } from './CommonDatabase';
import { Database } from './types';
+import { v4 as uuidv4 } from 'uuid';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-catalog-backend',
@@ -45,33 +46,62 @@ export class DatabaseManager {
return new CommonDatabase(knex, logger);
}
- public static async createInMemoryDatabase(
- options: Partial = {},
- ): Promise {
+ public static async createInMemoryDatabase(): Promise {
+ const knex = await this.createInMemoryDatabaseConnection();
+ return await this.createDatabase(knex);
+ }
+
+ public static async createInMemoryDatabaseConnection(): Promise {
const knex = Knex({
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
});
+
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
- return DatabaseManager.createDatabase(knex, options);
+
+ return knex;
}
public static async createTestDatabase(): Promise {
- const knex = Knex({
+ const knex = await this.createTestDatabaseConnection();
+ return await this.createDatabase(knex);
+ }
+
+ public static async createTestDatabaseConnection(): Promise {
+ const config: Knex.Config = {
+ /*
+ client: 'pg',
+ connection: {
+ host: 'localhost',
+ user: 'postgres',
+ password: 'postgres',
+ },
+ */
client: 'sqlite3',
connection: ':memory:',
useNullAsDefault: true,
- });
+ };
+
+ let knex = Knex(config);
+ if (typeof config.connection !== 'string') {
+ const tempDbName = `d${uuidv4().replace(/-/g, '')}`;
+ await knex.raw(`CREATE DATABASE ${tempDbName};`);
+ knex = Knex({
+ ...config,
+ connection: {
+ ...config.connection,
+ database: tempDbName,
+ },
+ });
+ }
+
knex.client.pool.on('createSuccess', (_eventId: any, resource: any) => {
resource.run('PRAGMA foreign_keys = ON', () => {});
});
- await knex.migrate.latest({
- directory: migrationsDir,
- });
- const { logger } = defaultOptions;
- return new CommonDatabase(knex, logger);
+
+ return knex;
}
}
diff --git a/plugins/catalog-backend/src/database/search.test.ts b/plugins/catalog-backend/src/database/search.test.ts
index fad17da733..c6793cbdf4 100644
--- a/plugins/catalog-backend/src/database/search.test.ts
+++ b/plugins/catalog-backend/src/database/search.test.ts
@@ -14,51 +14,39 @@
* limitations under the License.
*/
-import { ENTITY_DEFAULT_NAMESPACE, Entity } from '@backstage/catalog-model';
-import { buildEntitySearch, visitEntityPart } from './search';
-import type { DbEntitiesSearchRow } from './types';
+import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
+import { buildEntitySearch, mapToRows, traverse } from './search';
describe('search', () => {
- describe('visitEntityPart', () => {
+ describe('traverse', () => {
it('expands lists of strings to several rows', () => {
const input = { a: ['b', 'c', 'd'] };
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
+ const output = traverse(input);
expect(output).toEqual([
- { entity_id: 'eid', key: 'a', value: 'b' },
- { entity_id: 'eid', key: 'a', value: 'c' },
- { entity_id: 'eid', key: 'a', value: 'd' },
+ { key: 'a', value: 'b' },
+ { key: 'a.b', value: true },
+ { key: 'a', value: 'c' },
+ { key: 'a.c', value: true },
+ { key: 'a', value: 'd' },
+ { key: 'a.d', value: true },
]);
});
it('expands objects', () => {
const input = { a: { b: { c: 'd' }, e: 'f' } };
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
+ const output = traverse(input);
expect(output).toEqual([
- { entity_id: 'eid', key: 'a.b.c', value: 'd' },
- { entity_id: 'eid', key: 'a.e', value: 'f' },
+ { key: 'a.b.c', value: 'd' },
+ { key: 'a.e', value: 'f' },
]);
});
- it('converts base types to strings or null', () => {
- const input = {
- a: true,
- b: false,
- c: 7,
- d: 'string',
- e: null,
- f: undefined,
- };
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
+ it('expands list of objects', () => {
+ const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
+ const output = traverse(input);
expect(output).toEqual([
- { entity_id: 'eid', key: 'a', value: 'true' },
- { entity_id: 'eid', key: 'b', value: 'false' },
- { entity_id: 'eid', key: 'c', value: '7' },
- { entity_id: 'eid', key: 'd', value: 'string' },
- { entity_id: 'eid', key: 'e', value: null },
- { entity_id: 'eid', key: 'f', value: null },
+ { key: 'root.list.a', value: 1 },
+ { key: 'root.list.a', value: 2 },
]);
});
@@ -76,34 +64,47 @@ describe('search', () => {
},
d: 'd',
};
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
+ const output = traverse(input);
expect(output).toEqual([
- { entity_id: 'eid', key: 'a', value: 'a' },
- { entity_id: 'eid', key: 'metadata.b', value: 'b' },
- { entity_id: 'eid', key: 'metadata.c', value: 'c' },
- { entity_id: 'eid', key: 'd', value: 'd' },
+ { key: 'a', value: 'a' },
+ { key: 'metadata.b', value: 'b' },
+ { key: 'metadata.c', value: 'c' },
+ { key: 'd', value: 'd' },
]);
});
+ });
- it('expands list of objects', () => {
- const input = { root: { list: [{ a: 1 }, { a: 2 }] } };
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
+ describe('mapToRows', () => {
+ it('converts base types to strings or null', () => {
+ const input = [
+ { key: 'a', value: true },
+ { key: 'b', value: false },
+ { key: 'c', value: 7 },
+ { key: 'd', value: 'string' },
+ { key: 'e', value: null },
+ { key: 'f', value: undefined },
+ ];
+ const output = mapToRows(input, 'eid');
expect(output).toEqual([
- { entity_id: 'eid', key: 'root.list.a', value: '1' },
- { entity_id: 'eid', key: 'root.list.a', value: '2' },
+ { entity_id: 'eid', key: 'a', value: 'true' },
+ { entity_id: 'eid', key: 'b', value: 'false' },
+ { entity_id: 'eid', key: 'c', value: '7' },
+ { entity_id: 'eid', key: 'd', value: 'string' },
+ { entity_id: 'eid', key: 'e', value: null },
+ { entity_id: 'eid', key: 'f', value: null },
]);
});
it('emits lowercase version of keys and values', () => {
- const input = { theRoot: { listItems: [{ a: 'One' }, { a: 2 }] } };
- const output: DbEntitiesSearchRow[] = [];
- visitEntityPart('eid', '', input, output);
- expect(output).toEqual([
- { entity_id: 'eid', key: 'theroot.listitems.a', value: 'one' },
- { entity_id: 'eid', key: 'theroot.listitems.a', value: '2' },
- ]);
+ const input = [{ key: 'fOo', value: 'BaR' }];
+ const output = mapToRows(input, 'eid');
+ expect(output).toEqual([{ entity_id: 'eid', key: 'foo', value: 'bar' }]);
+ });
+
+ it('skips very large values', () => {
+ const input = [{ key: 'foo', value: 'a'.repeat(10000) }];
+ const output = mapToRows(input, 'eid');
+ expect(output).toEqual([]);
});
});
@@ -115,6 +116,8 @@ describe('search', () => {
metadata: { name: 'n' },
};
expect(buildEntitySearch('eid', input)).toEqual([
+ { entity_id: 'eid', key: 'apiversion', value: 'a' },
+ { entity_id: 'eid', key: 'kind', value: 'b' },
{ entity_id: 'eid', key: 'metadata.name', value: 'n' },
{ entity_id: 'eid', key: 'metadata.namespace', value: null },
{ entity_id: 'eid', key: 'metadata.uid', value: null },
@@ -123,8 +126,6 @@ describe('search', () => {
key: 'metadata.namespace',
value: ENTITY_DEFAULT_NAMESPACE,
},
- { entity_id: 'eid', key: 'apiversion', value: 'a' },
- { entity_id: 'eid', key: 'kind', value: 'b' },
]);
});
});
diff --git a/plugins/catalog-backend/src/database/search.ts b/plugins/catalog-backend/src/database/search.ts
index bbed2fac47..9e58d2468b 100644
--- a/plugins/catalog-backend/src/database/search.ts
+++ b/plugins/catalog-backend/src/database/search.ts
@@ -28,16 +28,18 @@ const SPECIAL_KEYS = [
'metadata.generation',
];
-function toValue(current: any): string | null {
- if (current === undefined || current === null) {
- return null;
- }
+// The maximum length allowed for search values. These columns are indexed, and
+// database engines do not like to index on massive values. For example,
+// postgres will balk after 8191 byte line sizes.
+const MAX_VALUE_LENGTH = 200;
- return String(current).toLowerCase();
-}
+type Kv = {
+ key: string;
+ value: unknown;
+};
-// Helper for iterating through a nested structure and outputting a list of
-// path->value entries.
+// Helper for traversing through a nested structure and outputting a list of
+// path->value entries of the leaves.
//
// For example, this yaml structure
//
@@ -53,57 +55,94 @@ function toValue(current: any): string | null {
//
// will result in
//
-// "a", "1"
+// "a", 1
// "b.c", null
// "b.e": "f"
+// "b.e.f": true
// "b.e": "g"
-// "h.i": "1"
+// "b.e.g": true
+// "h.i": 1
// "h.j": "k"
-// "h.i": "2"
+// "h.i": 2
// "h.j": "l"
-export function visitEntityPart(
- entityId: string,
- path: string,
- current: any,
- output: DbEntitiesSearchRow[],
-) {
- // ignored
- if (SPECIAL_KEYS.includes(path)) {
- return;
- }
+export function traverse(root: unknown): Kv[] {
+ const output: Kv[] = [];
- // empty or scalar
- if (
- current === undefined ||
- current === null ||
- ['string', 'number', 'boolean'].includes(typeof current)
- ) {
- output.push({ entity_id: entityId, key: path, value: toValue(current) });
- return;
- }
-
- // unknown
- if (typeof current !== 'object') {
- return;
- }
-
- // array
- if (Array.isArray(current)) {
- for (const item of current) {
- visitEntityPart(entityId, path, item, output);
+ function visit(path: string, current: unknown) {
+ if (SPECIAL_KEYS.includes(path)) {
+ return;
+ }
+
+ // empty or scalar
+ if (
+ current === undefined ||
+ current === null ||
+ ['string', 'number', 'boolean'].includes(typeof current)
+ ) {
+ output.push({ key: path, value: current });
+ return;
+ }
+
+ // unknown
+ if (typeof current !== 'object') {
+ return;
+ }
+
+ // array
+ if (Array.isArray(current)) {
+ for (const item of current) {
+ // NOTE(freben): The reason that these are output in two different ways,
+ // is to support use cases where you want to express that MORE than one
+ // tag is present in a list. Since the EntityFilters structure is a
+ // record, you can't have several entries of the same key. Therefore
+ // you will have to match on
+ //
+ // { "a.b": ["true"], "a.c": ["true"] }
+ //
+ // rather than
+ //
+ // { "a": ["b", "c"] }
+ //
+ // because the latter means EITHER b or c has to be present.
+ visit(path, item);
+ if (typeof item === 'string') {
+ output.push({ key: `${path}.${item}`, value: true });
+ }
+ }
+ return;
+ }
+
+ // object
+ for (const [key, value] of Object.entries(current!)) {
+ visit(path ? `${path}.${key}` : key, value);
}
- return;
}
- // object
- for (const [key, value] of Object.entries(current)) {
- visitEntityPart(
- entityId,
- (path ? `${path}.${key}` : key).toLowerCase(),
- value,
- output,
- );
+ visit('', root);
+
+ return output;
+}
+
+// Translates a number of raw data rows to search table rows
+export function mapToRows(
+ input: Kv[],
+ entityId: string,
+): DbEntitiesSearchRow[] {
+ const result: DbEntitiesSearchRow[] = [];
+
+ for (const { key: rawKey, value: rawValue } of input) {
+ const key = rawKey.toLowerCase();
+ if (rawValue === undefined || rawValue === null) {
+ result.push({ entity_id: entityId, key, value: null });
+ } else {
+ const value = String(rawValue).toLowerCase();
+ if (value.length <= MAX_VALUE_LENGTH) {
+ result.push({ entity_id: entityId, key, value });
+ }
+ }
}
+
+ return result;
}
/**
@@ -117,38 +156,20 @@ export function buildEntitySearch(
entityId: string,
entity: Entity,
): DbEntitiesSearchRow[] {
+ // Visit the entire structure recursively
+ const raw = traverse(entity);
+
// Start with some special keys that are always present because you want to
// be able to easily search for null specifically
- const result: DbEntitiesSearchRow[] = [
- {
- entity_id: entityId,
- key: 'metadata.name',
- value: toValue(entity.metadata.name),
- },
- {
- entity_id: entityId,
- key: 'metadata.namespace',
- value: toValue(entity.metadata.namespace),
- },
- {
- entity_id: entityId,
- key: 'metadata.uid',
- value: toValue(entity.metadata.uid),
- },
- ];
+ raw.push({ key: 'metadata.name', value: entity.metadata.name });
+ raw.push({ key: 'metadata.namespace', value: entity.metadata.namespace });
+ raw.push({ key: 'metadata.uid', value: entity.metadata.uid });
// Namespace not specified has the default value "default", so we want to
// match on that as well
if (!entity.metadata.namespace) {
- result.push({
- entity_id: entityId,
- key: 'metadata.namespace',
- value: toValue(ENTITY_DEFAULT_NAMESPACE),
- });
+ raw.push({ key: 'metadata.namespace', value: ENTITY_DEFAULT_NAMESPACE });
}
- // Visit the entire structure recursively
- visitEntityPart(entityId, '', entity, result);
-
- return result;
+ return mapToRows(raw, entityId);
}
diff --git a/plugins/catalog-backend/src/database/types.ts b/plugins/catalog-backend/src/database/types.ts
index c0f5777743..b8a2a6231a 100644
--- a/plugins/catalog-backend/src/database/types.ts
+++ b/plugins/catalog-backend/src/database/types.ts
@@ -14,20 +14,20 @@
* limitations under the License.
*/
-import type { Entity, EntityName, Location } from '@backstage/catalog-model';
+import type {
+ Entity,
+ EntityName,
+ Location,
+ EntityRelationSpec,
+} from '@backstage/catalog-model';
export type DbEntitiesRow = {
id: string;
location_id: string | null;
- api_version: string;
- kind: string;
- name: string | null;
- namespace: string | null;
etag: string;
generation: number;
full_name: string;
- metadata: string;
- spec: string | null;
+ data: string;
};
export type DbEntityRequest = {
@@ -40,6 +40,13 @@ export type DbEntityResponse = {
entity: Entity;
};
+export type DbEntitiesRelationsRow = {
+ originating_entity_id: string;
+ source_full_name: string;
+ type: string;
+ target_full_name: string;
+};
+
export type DbEntitiesSearchRow = {
entity_id: string;
key: string;
@@ -72,11 +79,27 @@ export type DatabaseLocationUpdateLogEvent = {
message?: string;
};
-export type EntityFilter = {
- key: string;
- values: (string | null)[];
-};
-export type EntityFilters = EntityFilter[];
+/**
+ * Filter matcher for a single entity field.
+ *
+ * Can be either null or a string, or an array of those. Null and the empty
+ * string are treated equally, and match both a present field with a null or
+ * empty value, as well as an absent field.
+ *
+ * A filter may contain asterisks (*) that are treated as wildcards for zero
+ * or more arbitrary characters.
+ */
+export type EntityFilter = null | string | (null | string)[];
+
+/**
+ * A set of filter matchers used for filtering entities.
+ *
+ * The keys are full dot-separated paths into the structure of an entity, for
+ * example "metadata.name". You can also address any item in an array the same
+ * way, e.g. "a.b.c": "x" works if b is an array of objects that have a c field
+ * and any of those have the value x.
+ */
+export type EntityFilters = Record;
/**
* An abstraction on top of the underlying database, wrapping the basic CRUD
@@ -94,13 +117,15 @@ export type Database = {
transaction(fn: (tx: unknown) => Promise): Promise;
/**
- * Adds a new entity to the catalog.
+ * Adds a set of new entities to the catalog.
*
* @param tx An ongoing transaction
- * @param request The entity being added
- * @returns The added entity, with uid, etag and generation set
+ * @param request The entities being added
*/
- addEntity(tx: unknown, request: DbEntityRequest): Promise;
+ addEntities(
+ tx: unknown,
+ request: DbEntityRequest[],
+ ): Promise;
/**
* Updates an existing entity in the catalog.
@@ -128,7 +153,7 @@ export type Database = {
matchingGeneration?: number,
): Promise;
- entities(tx: unknown, filters?: EntityFilters): Promise;
+ entities(tx: unknown, filters?: EntityFilters[]): Promise;
entityByName(
tx: unknown,
@@ -137,7 +162,19 @@ export type Database = {
entityByUid(tx: unknown, uid: string): Promise;
- removeEntity(tx: unknown, uid: string): Promise;
+ removeEntityByUid(tx: unknown, uid: string): Promise;
+
+ /**
+ * Remove current relations for the entity and replace them with the new relations array
+ * @param tx An ongoing transaction
+ * @param entityUid the entity uid
+ * @param relations the relationships to be set
+ */
+ setRelations(
+ tx: unknown,
+ entityUid: string,
+ relations: EntityRelationSpec[],
+ ): Promise;
addLocation(location: Location): Promise;
diff --git a/plugins/catalog-backend/src/index.ts b/plugins/catalog-backend/src/index.ts
index 89fd4d37ed..f621930d10 100644
--- a/plugins/catalog-backend/src/index.ts
+++ b/plugins/catalog-backend/src/index.ts
@@ -17,5 +17,5 @@
export * from './catalog';
export * from './database';
export * from './ingestion';
-export * from './service/router';
+export * from './service';
export * from './util';
diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
index 32fcc27909..933433b47e 100644
--- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
+++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
@@ -15,12 +15,7 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
-import {
- Entity,
- ENTITY_DEFAULT_NAMESPACE,
- Location,
- LocationSpec,
-} from '@backstage/catalog-model';
+import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { LocationUpdateStatus } from '../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../database/types';
@@ -36,10 +31,8 @@ describe('HigherOrderOperations', () => {
beforeAll(() => {
entitiesCatalog = {
entities: jest.fn(),
- entityByUid: jest.fn(),
- entityByName: jest.fn(),
- addOrUpdateEntity: jest.fn(),
removeEntityByUid: jest.fn(),
+ batchAddOrUpdateEntities: jest.fn(),
};
locationsCatalog = {
addLocation: jest.fn(),
@@ -73,7 +66,10 @@ describe('HigherOrderOperations', () => {
};
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
locationsCatalog.locations.mockResolvedValue([]);
- locationReader.read.mockResolvedValue({ entities: [], errors: [] });
+ locationReader.read.mockResolvedValue({
+ entities: [],
+ errors: [],
+ });
const result = await higherOrderOperation.addLocation(spec);
@@ -87,7 +83,7 @@ describe('HigherOrderOperations', () => {
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
- expect(entitiesCatalog.addOrUpdateEntity).not.toBeCalled();
+ expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
expect(locationsCatalog.addLocation).toBeCalledWith(
expect.objectContaining({
@@ -113,7 +109,10 @@ describe('HigherOrderOperations', () => {
data: location,
},
]);
- locationReader.read.mockResolvedValue({ entities: [], errors: [] });
+ locationReader.read.mockResolvedValue({
+ entities: [],
+ errors: [],
+ });
const result = await higherOrderOperation.addLocation(spec);
@@ -122,7 +121,7 @@ describe('HigherOrderOperations', () => {
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
- expect(entitiesCatalog.addOrUpdateEntity).not.toBeCalled();
+ expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).not.toBeCalled();
});
@@ -140,7 +139,7 @@ describe('HigherOrderOperations', () => {
locationsCatalog.locations.mockResolvedValue([]);
locationReader.read.mockResolvedValue({
- entities: [{ entity, location }],
+ entities: [{ entity, location, relations: [] }],
errors: [{ error: new Error('abcd'), location }],
});
@@ -148,7 +147,7 @@ describe('HigherOrderOperations', () => {
/abcd/,
);
expect(locationsCatalog.locations).toBeCalledTimes(1);
- expect(entitiesCatalog.addOrUpdateEntity).not.toBeCalled();
+ expect(entitiesCatalog.batchAddOrUpdateEntities).not.toBeCalled();
expect(locationsCatalog.addLocation).not.toBeCalled();
});
});
@@ -163,7 +162,7 @@ describe('HigherOrderOperations', () => {
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(locationReader.read).not.toHaveBeenCalled();
- expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled();
+ expect(entitiesCatalog.batchAddOrUpdateEntities).not.toHaveBeenCalled();
});
it('can update a single location where a matching entity did not exist', async () => {
@@ -183,16 +182,18 @@ describe('HigherOrderOperations', () => {
metadata: { name: 'c1' },
spec: { type: 'service' },
};
+ const entityId = 'xyz123';
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
locationReader.read.mockResolvedValue({
- entities: [{ entity: desc, location }],
+ entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
- entitiesCatalog.entityByName.mockResolvedValue(undefined);
- entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
+ entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([
+ { entityId },
+ ]);
await expect(
higherOrderOperation.refreshAllLocations(),
@@ -204,18 +205,14 @@ describe('HigherOrderOperations', () => {
type: 'some',
target: 'thing',
});
- expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
- expect(entitiesCatalog.entityByName).toHaveBeenNthCalledWith(1, {
- kind: 'Component',
- namespace: ENTITY_DEFAULT_NAMESPACE,
- name: 'c1',
- });
- expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1);
- expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith(
- 1,
- expect.objectContaining({
- metadata: expect.objectContaining({ name: 'c1' }),
- }),
+ expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledTimes(1);
+ expect(entitiesCatalog.batchAddOrUpdateEntities).toHaveBeenCalledWith(
+ [
+ expect.objectContaining({
+ entity: expect.objectContaining({ metadata: { name: 'c1' } }),
+ relations: [],
+ }),
+ ],
'123',
);
});
@@ -242,11 +239,11 @@ describe('HigherOrderOperations', () => {
{ currentStatus: locationStatus, data: location },
]);
locationReader.read.mockResolvedValue({
- entities: [{ entity: desc, location }],
+ entities: [{ entity: desc, location, relations: [] }],
errors: [],
});
- entitiesCatalog.entityByName.mockResolvedValue(undefined);
- entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
+ entitiesCatalog.entities.mockResolvedValue([]);
+ entitiesCatalog.batchAddOrUpdateEntities.mockResolvedValue([]);
await expect(
higherOrderOperation.refreshAllLocations(),
diff --git a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
index 571602b966..0e62815be5 100644
--- a/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
+++ b/plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
@@ -14,18 +14,11 @@
* limitations under the License.
*/
-import { InputError } from '@backstage/backend-common';
-import {
- Entity,
- entityHasChanges,
- getEntityName,
- Location,
- LocationSpec,
- serializeEntityRef,
-} from '@backstage/catalog-model';
+import { Location, LocationSpec } from '@backstage/catalog-model';
import { v4 as uuidv4 } from 'uuid';
import { Logger } from 'winston';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
+import { durationText } from '../util/timing';
import {
AddLocationResult,
HigherOrderOperation,
@@ -85,11 +78,9 @@ export class HigherOrderOperations implements HigherOrderOperation {
// Read the location fully, bailing on any errors
const readerOutput = await this.locationReader.read(spec);
- if (readerOutput.errors.length) {
+ if (!(spec.presence === 'optional') && readerOutput.errors.length) {
const item = readerOutput.errors[0];
- throw new InputError(
- `Failed to read location ${item.location.type} ${item.location.target}, ${item.error}`,
- );
+ throw item.error;
}
// TODO(freben): At this point, we could detect orphaned entities, by way
@@ -100,16 +91,20 @@ export class HigherOrderOperations implements HigherOrderOperation {
if (!previousLocation) {
await this.locationsCatalog.addLocation(location);
}
- const outputEntities: Entity[] = [];
- for (const entity of readerOutput.entities) {
- const out = await this.entitiesCatalog.addOrUpdateEntity(
- entity.entity,
- location.id,
- );
- outputEntities.push(out);
+ if (readerOutput.entities.length === 0) {
+ return { location, entities: [] };
}
- return { location, entities: outputEntities };
+ const writtenEntities = await this.entitiesCatalog.batchAddOrUpdateEntities(
+ readerOutput.entities,
+ location.id,
+ );
+
+ const entities = await this.entitiesCatalog.entities([
+ { 'metadata.uid': writtenEntities.map(e => e.entityId) },
+ ]);
+
+ return { location, entities };
}
/**
@@ -121,85 +116,84 @@ export class HigherOrderOperations implements HigherOrderOperation {
* without changes.
*/
async refreshAllLocations(): Promise {
- const startTimestamp = new Date().valueOf();
+ const startTimestamp = process.hrtime();
this.logger.info('Beginning locations refresh');
const locations = await this.locationsCatalog.locations();
this.logger.info(`Visiting ${locations.length} locations`);
for (const { data: location } of locations) {
- this.logger.debug(
- `Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
+ this.logger.info(
+ `Refreshing location ${location.type}:${location.target}`,
);
try {
await this.refreshSingleLocation(location);
await this.locationsCatalog.logUpdateSuccess(location.id, undefined);
} catch (e) {
- this.logger.debug(
- `Failed to refresh location id="${location.id}" type="${location.type}" target="${location.target}", ${e}`,
+ this.logger.warn(
+ `Failed to refresh location ${location.type}:${location.target}, ${e}`,
);
await this.locationsCatalog.logUpdateFailure(location.id, e);
}
}
- const endTimestamp = new Date().valueOf();
- const duration = ((endTimestamp - startTimestamp) / 1000).toFixed(1);
- this.logger.debug(`Completed locations refresh in ${duration} seconds`);
+ this.logger.info(
+ `Completed locations refresh in ${durationText(startTimestamp)}`,
+ );
}
// Performs a full refresh of a single location
private async refreshSingleLocation(location: Location) {
+ let startTimestamp = process.hrtime();
+
const readerOutput = await this.locationReader.read({
type: location.type,
target: location.target,
});
for (const item of readerOutput.errors) {
- this.logger.debug(
- `Failed item in location type="${item.location.type}" target="${item.location.target}", ${item.error}`,
+ this.logger.warn(
+ `Failed item in location ${item.location.type}:${item.location.target}, ${item.error}`,
);
}
this.logger.info(
- `Read ${readerOutput.entities.length} entities from location ${location.type} ${location.target}`,
+ `Read ${readerOutput.entities.length} entities from location ${
+ location.type
+ }:${location.target} in ${durationText(startTimestamp)}`,
);
- const startTimestamp = process.hrtime();
- for (const item of readerOutput.entities) {
- const { entity } = item;
-
- try {
- const previous = await this.entitiesCatalog.entityByName(
- getEntityName(entity),
- );
-
- if (!previous) {
- await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
- } else if (entityHasChanges(previous, entity)) {
- await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
- }
-
- await this.locationsCatalog.logUpdateSuccess(
- location.id,
- entity.metadata.name,
- );
- } catch (error) {
- this.logger.info(
- `Failed refresh of entity ${serializeEntityRef(entity)}, ${error}`,
- );
+ startTimestamp = process.hrtime();
+ try {
+ await this.entitiesCatalog.batchAddOrUpdateEntities(
+ readerOutput.entities,
+ location.id,
+ );
+ } catch (e) {
+ for (const entity of readerOutput.entities) {
await this.locationsCatalog.logUpdateFailure(
location.id,
- error,
- entity.metadata.name,
+ e,
+ entity.entity.metadata.name,
);
}
+ throw e;
+ }
+
+ this.logger.info(`Posting update success markers`);
+
+ for (const entity of readerOutput.entities) {
+ await this.locationsCatalog.logUpdateSuccess(
+ location.id,
+ entity.entity.metadata.name,
+ );
}
- const delta = process.hrtime(startTimestamp);
- const durationMs = ((delta[0] * 1e9 + delta[1]) / 1e6).toFixed(1);
this.logger.info(
- `Wrote ${readerOutput.entities.length} entities from location ${location.type} ${location.target} in ${durationMs} seconds`,
+ `Wrote ${readerOutput.entities.length} entities from location ${
+ location.type
+ }:${location.target} in ${durationText(startTimestamp)}`,
);
}
}
diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts
index fcc608eb51..45bf2e8ae0 100644
--- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts
+++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts
@@ -14,44 +14,26 @@
* limitations under the License.
*/
-import { getVoidLogger, UrlReader } from '@backstage/backend-common';
+import { UrlReader } from '@backstage/backend-common';
import {
Entity,
- EntityPolicies,
EntityPolicy,
+ EntityRelationSpec,
ENTITY_DEFAULT_NAMESPACE,
LocationSpec,
} from '@backstage/catalog-model';
-import { Config, ConfigReader } from '@backstage/config';
+import { Config } from '@backstage/config';
import { Logger } from 'winston';
import { CatalogRulesEnforcer } from './CatalogRules';
-import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor';
-import { ApiDefinitionAtLocationProcessor } from './processors/ApiDefinitionAtLocationProcessor';
-import { AzureApiReaderProcessor } from './processors/AzureApiReaderProcessor';
-import { BitbucketApiReaderProcessor } from './processors/BitbucketApiReaderProcessor';
-import { CodeOwnersProcessor } from './processors/CodeOwnersProcessor';
-import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
-import { FileReaderProcessor } from './processors/FileReaderProcessor';
-import { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
-import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
-import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor';
-import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor';
-import { LdapOrgReaderProcessor } from './processors/LdapOrgReaderProcessor';
-import { LocationRefProcessor } from './processors/LocationEntityProcessor';
-import { PlaceholderProcessor } from './processors/PlaceholderProcessor';
import * as result from './processors/results';
-import { StaticLocationProcessor } from './processors/StaticLocationProcessor';
import {
- LocationProcessor,
- LocationProcessorDataResult,
- LocationProcessorEmit,
- LocationProcessorEntityResult,
- LocationProcessorErrorResult,
- LocationProcessorLocationResult,
- LocationProcessorResult,
+ CatalogProcessor,
+ CatalogProcessorEmit,
+ CatalogProcessorEntityResult,
+ CatalogProcessorErrorResult,
+ CatalogProcessorLocationResult,
+ CatalogProcessorResult,
} from './processors/types';
-import { UrlReaderProcessor } from './processors/UrlReaderProcessor';
-import { YamlProcessor } from './processors/YamlProcessor';
import { LocationReader, ReadLocationResult } from './types';
// The max amount of nesting depth of generated work items
@@ -59,114 +41,58 @@ const MAX_DEPTH = 10;
type Options = {
reader: UrlReader;
- logger?: Logger;
- config?: Config;
- processors?: LocationProcessor[];
+ logger: Logger;
+ config: Config;
+ processors: CatalogProcessor[];
+ rulesEnforcer: CatalogRulesEnforcer;
+ policy: EntityPolicy;
};
/**
* Implements the reading of a location through a series of processor tasks.
*/
export class LocationReaders implements LocationReader {
- private readonly logger: Logger;
- private readonly processors: LocationProcessor[];
- private readonly rulesEnforcer: CatalogRulesEnforcer;
+ private readonly options: Options;
- static defaultProcessors(options: {
- logger: Logger;
- reader: UrlReader;
- config?: Config;
- entityPolicy?: EntityPolicy;
- }): LocationProcessor[] {
- const {
- logger,
- config = new ConfigReader({}, 'missing-config'),
- entityPolicy = new EntityPolicies(),
- } = options;
-
- // TODO(Rugvip): These are added for backwards compatibility if config exists
- // The idea is to have everyone migrate from using the old processors to the new
- // integration config driven UrlReaders. In an upcoming release we can then completely
- // remove support for the old processors, but still keep handling the deprecated location
- // types for a while, but with a warning.
- const oldProcessors = [];
- const pc = config.getOptionalConfig('catalog.processors');
- if (pc?.has('github')) {
- logger.warn(
- `Using deprecated configuration for catalog.processors.github, move to using integrations.github instead`,
- );
- oldProcessors.push(GithubReaderProcessor.fromConfig(config, logger));
- }
- if (pc?.has('gitlabApi')) {
- logger.warn(
- `Using deprecated configuration for catalog.processors.gitlabApi, move to using integrations.gitlab instead`,
- );
- oldProcessors.push(new GitlabApiReaderProcessor(config));
- oldProcessors.push(new GitlabReaderProcessor());
- }
- if (pc?.has('bitbucketApi')) {
- logger.warn(
- `Using deprecated configuration for catalog.processors.bitbucketApi, move to using integrations.bitbucket instead`,
- );
- oldProcessors.push(new BitbucketApiReaderProcessor(config));
- }
- if (pc?.has('azureApi')) {
- logger.warn(
- `Using deprecated configuration for catalog.processors.azureApi, move to using integrations.azure instead`,
- );
- oldProcessors.push(new AzureApiReaderProcessor(config));
- }
-
- return [
- StaticLocationProcessor.fromConfig(config),
- new FileReaderProcessor(),
- ...oldProcessors,
- GithubOrgReaderProcessor.fromConfig(config, { logger }),
- LdapOrgReaderProcessor.fromConfig(config, { logger }),
- new UrlReaderProcessor(options),
- new YamlProcessor(),
- PlaceholderProcessor.default(),
- new CodeOwnersProcessor(),
- new ApiDefinitionAtLocationProcessor(),
- new EntityPolicyProcessor(entityPolicy),
- new LocationRefProcessor(),
- new AnnotateLocationEntityProcessor(),
- ];
- }
-
- constructor({
- logger = getVoidLogger(),
- config,
- reader,
- processors = LocationReaders.defaultProcessors({ logger, reader, config }),
- }: Options) {
- this.logger = logger;
- this.processors = processors;
- this.rulesEnforcer = config
- ? CatalogRulesEnforcer.fromConfig(config)
- : new CatalogRulesEnforcer(CatalogRulesEnforcer.defaultRules);
+ constructor(options: Options) {
+ this.options = options;
}
async read(location: LocationSpec): Promise {
- const output: ReadLocationResult = { entities: [], errors: [] };
- let items: LocationProcessorResult[] = [result.location(location, false)];
+ const { rulesEnforcer, logger } = this.options;
+
+ const output: ReadLocationResult = {
+ entities: [],
+ errors: [],
+ };
+ let items: CatalogProcessorResult[] = [result.location(location, false)];
for (let depth = 0; depth < MAX_DEPTH; ++depth) {
- const newItems: LocationProcessorResult[] = [];
- const emit: LocationProcessorEmit = i => newItems.push(i);
+ const newItems: CatalogProcessorResult[] = [];
+ const emit: CatalogProcessorEmit = i => newItems.push(i);
for (const item of items) {
if (item.type === 'location') {
await this.handleLocation(item, emit);
- } else if (item.type === 'data') {
- await this.handleData(item, emit);
} else if (item.type === 'entity') {
- if (this.rulesEnforcer.isAllowed(item.entity, item.location)) {
- const entity = await this.handleEntity(item, emit);
- output.entities.push({
- entity,
- location: item.location,
+ if (rulesEnforcer.isAllowed(item.entity, item.location)) {
+ const relations = Array();
+
+ const entity = await this.handleEntity(item, emitResult => {
+ if (emitResult.type === 'relation') {
+ relations.push(emitResult.relation);
+ return;
+ }
+ emit(emitResult);
});
+
+ if (entity) {
+ output.entities.push({
+ entity,
+ location: item.location,
+ relations,
+ });
+ }
} else {
output.errors.push({
location: item.location,
@@ -192,83 +118,112 @@ export class LocationReaders implements LocationReader {
}
const message = `Max recursion depth ${MAX_DEPTH} reached for ${location.type} ${location.target}`;
- this.logger.warn(message);
+ logger.warn(message);
output.errors.push({ location, error: new Error(message) });
return output;
}
private async handleLocation(
- item: LocationProcessorLocationResult,
- emit: LocationProcessorEmit,
+ item: CatalogProcessorLocationResult,
+ emit: CatalogProcessorEmit,
) {
- for (const processor of this.processors) {
+ const { processors, logger } = this.options;
+
+ const validatedEmit: CatalogProcessorEmit = emitResult => {
+ if (emitResult.type === 'relation') {
+ throw new Error('readLocation may not emit entity relations');
+ }
+
+ emit(emitResult);
+ };
+
+ for (const processor of processors) {
if (processor.readLocation) {
try {
if (
- await processor.readLocation(item.location, item.optional, emit)
+ await processor.readLocation(
+ item.location,
+ item.optional,
+ validatedEmit,
+ )
) {
return;
}
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while reading location ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
- this.logger.warn(message);
+ logger.warn(message);
}
}
}
const message = `No processor was able to read location ${item.location.type} ${item.location.target}`;
emit(result.inputError(item.location, message));
- this.logger.warn(message);
+ logger.warn(message);
}
- private async handleData(
- item: LocationProcessorDataResult,
- emit: LocationProcessorEmit,
- ) {
- for (const processor of this.processors) {
- if (processor.parseData) {
+ private async handleEntity(
+ item: CatalogProcessorEntityResult,
+ emit: CatalogProcessorEmit,
+ ): Promise {
+ const { processors, logger } = this.options;
+
+ let current = item.entity;
+
+ // Construct the name carefully, this happens before validation below
+ // so we do not want to crash here due to missing metadata or so
+ const kind = current.kind || '';
+ const namespace = !current.metadata
+ ? ''
+ : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
+ const name = !current.metadata ? '' : current.metadata.name;
+
+ for (const processor of processors) {
+ if (processor.preProcessEntity) {
try {
- if (await processor.parseData(item.data, item.location, emit)) {
- return;
- }
+ current = await processor.preProcessEntity(
+ current,
+ item.location,
+ emit,
+ );
} catch (e) {
- const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`;
- emit(result.generalError(item.location, message));
- this.logger.warn(message);
+ const message = `Processor ${processor.constructor.name} threw an error while preprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
+ emit(result.generalError(item.location, e.message));
+ logger.warn(message);
+ return undefined;
}
}
}
- const message = `No processor was able to parse location ${item.location.type} ${item.location.target}`;
- emit(result.inputError(item.location, message));
- }
+ try {
+ const next = await this.options.policy.enforce(current);
+ if (!next) {
+ const message = `Policy unexpectedly returned no data while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}`;
+ emit(result.generalError(item.location, message));
+ logger.warn(message);
+ return undefined;
+ }
+ current = next;
+ } catch (e) {
+ const message = `Policy check failed while analyzing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
+ emit(result.inputError(item.location, e.message));
+ logger.warn(message);
+ return undefined;
+ }
- private async handleEntity(
- item: LocationProcessorEntityResult,
- emit: LocationProcessorEmit,
- ): Promise {
- let current = item.entity;
-
- for (const processor of this.processors) {
- if (processor.processEntity) {
+ for (const processor of processors) {
+ if (processor.postProcessEntity) {
try {
- current = await processor.processEntity(
+ current = await processor.postProcessEntity(
current,
item.location,
emit,
- this.readLocation.bind(this),
);
} catch (e) {
- // Construct the name carefully, if we got validation errors we do
- // not want to crash here due to missing metadata or so
- const namespace = !current.metadata
- ? ''
- : current.metadata.namespace ?? ENTITY_DEFAULT_NAMESPACE;
- const name = !current.metadata ? '' : current.metadata.name;
- const message = `Processor ${processor.constructor.name} threw an error while processing entity ${current.kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
+ const message = `Processor ${processor.constructor.name} threw an error while postprocessing entity ${kind}:${namespace}/${name} at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
- this.logger.warn(message);
+ logger.warn(message);
+ return undefined;
}
}
}
@@ -277,59 +232,33 @@ export class LocationReaders implements LocationReader {
}
private async handleError(
- item: LocationProcessorErrorResult,
- emit: LocationProcessorEmit,
+ item: CatalogProcessorErrorResult,
+ emit: CatalogProcessorEmit,
) {
- this.logger.debug(
+ const { processors, logger } = this.options;
+
+ logger.debug(
`Encountered error at location ${item.location.type} ${item.location.target}, ${item.error}`,
);
- for (const processor of this.processors) {
+ const validatedEmit: CatalogProcessorEmit = emitResult => {
+ if (emitResult.type === 'relation') {
+ throw new Error('handleError may not emit entity relations');
+ }
+
+ emit(emitResult);
+ };
+
+ for (const processor of processors) {
if (processor.handleError) {
try {
- await processor.handleError(item.error, item.location, emit);
+ await processor.handleError(item.error, item.location, validatedEmit);
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while handling another error at ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
- this.logger.warn(message);
+ logger.warn(message);
}
}
}
}
-
- private async readLocation(location: LocationSpec): Promise {
- let data: Buffer | undefined = undefined;
- let error: Error | undefined = undefined;
-
- await this.handleLocation(
- {
- type: 'location',
- location,
- optional: false,
- },
- output => {
- if (output.type === 'error' && !error) {
- error = output.error;
- } else if (output.type === 'data') {
- if (data) {
- if (!error) {
- error = new Error(
- 'More than one piece of data loaded unexpectedly',
- );
- }
- } else {
- data = output.data;
- }
- }
- },
- );
-
- if (error) {
- throw error;
- } else if (!data) {
- throw new Error('No data loaded');
- }
-
- return data;
- }
}
diff --git a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
index ec7e59281e..ea8afcddc5 100644
--- a/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
@@ -16,10 +16,13 @@
import { Entity, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
-import { LocationProcessor } from './types';
+import { CatalogProcessor } from './types';
-export class AnnotateLocationEntityProcessor implements LocationProcessor {
- async processEntity(entity: Entity, location: LocationSpec): Promise {
+export class AnnotateLocationEntityProcessor implements CatalogProcessor {
+ async preProcessEntity(
+ entity: Entity,
+ location: LocationSpec,
+ ): Promise {
return lodash.merge(
{
metadata: {
diff --git a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts
deleted file mode 100644
index 6ae6562339..0000000000
--- a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.test.ts
+++ /dev/null
@@ -1,93 +0,0 @@
-import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model';
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import { ApiDefinitionAtLocationProcessor } from './ApiDefinitionAtLocationProcessor';
-
-describe('ApiDefinitionAtLocationProcessor', () => {
- let processor: ApiDefinitionAtLocationProcessor;
- let entity: Entity;
- let location: LocationSpec;
-
- beforeEach(() => {
- processor = new ApiDefinitionAtLocationProcessor();
- entity = {
- apiVersion: 'backstage.io/v1alpha1',
- kind: 'API',
- metadata: {
- name: 'test',
- },
- spec: {
- lifecycle: 'production',
- owner: 'info@example.com',
- type: 'openapi',
- definition: 'Hello',
- },
- };
- location = {
- type: 'url',
- target: `http://example.com/api.yaml`,
- };
- });
-
- it('should skip entities without annotation', async () => {
- const read = jest.fn().mockRejectedValue(new Error('boo'));
-
- const generated = (await processor.processEntity(
- entity,
- location,
- () => {},
- read,
- )) as ApiEntity;
-
- expect(generated.spec.definition).toBe('Hello');
- });
-
- it('should load from location', async () => {
- entity.metadata.annotations = {
- 'backstage.io/definition-at-location':
- 'url:http://example.com/openapi.yaml',
- };
-
- const read = jest.fn().mockResolvedValue(Buffer.from('Hello'));
-
- const generated = (await processor.processEntity(
- entity,
- location,
- () => {},
- read,
- )) as ApiEntity;
-
- expect(generated.spec.definition).toBe('Hello');
- expect(read.mock.calls[0][0]).toStrictEqual({
- type: 'url',
- target: 'http://example.com/openapi.yaml',
- });
- });
-
- it('should throw errors while loading', async () => {
- entity.metadata.annotations = {
- 'backstage.io/definition-at-location': 'missing',
- };
-
- const read = jest
- .fn()
- .mockRejectedValue(new Error('Failed to load location'));
-
- await expect(
- processor.processEntity(entity, location, () => {}, read),
- ).rejects.toThrow('Failed to load location');
- });
-});
diff --git a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts
deleted file mode 100644
index e64d112e9c..0000000000
--- a/plugins/catalog-backend/src/ingestion/processors/ApiDefinitionAtLocationProcessor.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { ApiEntity, Entity, LocationSpec } from '@backstage/catalog-model';
-import {
- LocationProcessor,
- LocationProcessorEmit,
- LocationProcessorRead,
-} from './types';
-
-const DEFINITION_AT_LOCATION_ANNOTATION = 'backstage.io/definition-at-location';
-
-export class ApiDefinitionAtLocationProcessor implements LocationProcessor {
- async processEntity(
- entity: Entity,
- _location: LocationSpec,
- _emit: LocationProcessorEmit,
- read: LocationProcessorRead,
- ): Promise {
- if (
- entity.kind !== 'API' ||
- !entity.metadata.annotations ||
- !entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION]
- ) {
- return entity;
- }
-
- const reference =
- entity.metadata.annotations[DEFINITION_AT_LOCATION_ANNOTATION];
- const { type, target } = extractReference(reference);
- const data = await read({ type, target });
- const definition = data.toString();
- const apiEntity = entity as ApiEntity;
- apiEntity.spec.definition = definition;
-
- return entity;
- }
-}
-
-function extractReference(reference: string): { type: string; target: string } {
- const delimiterIndex = reference.indexOf(':');
- const type = reference.slice(0, delimiterIndex);
- const target = reference.slice(delimiterIndex + 1);
-
- return { type, target };
-}
diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
deleted file mode 100644
index 2a15c18c84..0000000000
--- a/plugins/catalog-backend/src/ingestion/processors/AzureApiReaderProcessor.ts
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { LocationSpec } from '@backstage/catalog-model';
-import fetch, { RequestInit, HeadersInit } from 'node-fetch';
-import * as result from './results';
-import { LocationProcessor, LocationProcessorEmit } from './types';
-import { Config } from '@backstage/config';
-
-// ***********************************************************************
-// * NOTE: This has been replaced by packages/backend-common/src/reading *
-// * Don't implement new functionality here as this file will be removed *
-// ***********************************************************************
-
-export class AzureApiReaderProcessor implements LocationProcessor {
- private privateToken: string;
-
- constructor(config: Config) {
- this.privateToken =
- config.getOptionalString('catalog.processors.azureApi.privateToken') ??
- '';
- }
-
- getRequestOptions(): RequestInit {
- const headers: HeadersInit = {};
-
- if (this.privateToken !== '') {
- headers.Authorization = `Basic ${Buffer.from(
- `:${this.privateToken}`,
- 'utf8',
- ).toString('base64')}`;
- }
-
- const requestOptions: RequestInit = {
- headers,
- };
-
- return requestOptions;
- }
-
- async readLocation(
- location: LocationSpec,
- optional: boolean,
- emit: LocationProcessorEmit,
- ): Promise {
- if (location.type !== 'azure/api') {
- return false;
- }
-
- try {
- const url = this.buildRawUrl(location.target);
-
- const response = await fetch(url.toString(), this.getRequestOptions());
-
- // for private repos when PAT is not valid, Azure API returns a http status code 203 with sign in page html
- if (response.ok && response.status !== 203) {
- const data = await response.buffer();
- emit(result.data(location, data));
- } else {
- const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
- if (response.status === 404) {
- if (!optional) {
- emit(result.notFoundError(location, message));
- }
- } else {
- emit(result.generalError(location, message));
- }
- }
- } catch (e) {
- const message = `Unable to read ${location.type} ${location.target}, ${e}`;
- emit(result.generalError(location, message));
- }
- return true;
- }
-
- // Converts
- // from: https://dev.azure.com/{organization}/{project}/_git/reponame?path={path}&version=GB{commitOrBranch}&_a=contents
- // to: https://dev.azure.com/{organization}/{project}/_apis/git/repositories/reponame/items?path={path}&version={commitOrBranch}
- buildRawUrl(target: string): URL {
- try {
- const url = new URL(target);
-
- const [
- empty,
- userOrOrg,
- project,
- srcKeyword,
- repoName,
- ] = url.pathname.split('/');
-
- const path = url.searchParams.get('path') || '';
- const ref = url.searchParams.get('version')?.substr(2);
-
- if (
- url.hostname !== 'dev.azure.com' ||
- empty !== '' ||
- userOrOrg === '' ||
- project === '' ||
- srcKeyword !== '_git' ||
- repoName === '' ||
- path === '' ||
- ref === ''
- ) {
- throw new Error('Wrong Azure Devops URL or Invalid file path');
- }
-
- // transform to api
- url.pathname = [
- empty,
- userOrOrg,
- project,
- '_apis',
- 'git',
- 'repositories',
- repoName,
- 'items',
- ].join('/');
-
- const queryParams = [`path=${path}`];
-
- if (ref) {
- queryParams.push(`version=${ref}`);
- }
-
- url.search = queryParams.join('&');
-
- url.protocol = 'https';
-
- return url;
- } catch (e) {
- throw new Error(`Incorrect url: ${target}, ${e}`);
- }
- }
-}
diff --git a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
deleted file mode 100644
index b3720a5b76..0000000000
--- a/plugins/catalog-backend/src/ingestion/processors/BitbucketApiReaderProcessor.ts
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { LocationSpec } from '@backstage/catalog-model';
-import fetch, { RequestInit, HeadersInit } from 'node-fetch';
-import * as result from './results';
-import { LocationProcessor, LocationProcessorEmit } from './types';
-import { Config } from '@backstage/config';
-
-// ***********************************************************************
-// * NOTE: This has been replaced by packages/backend-common/src/reading *
-// * Don't implement new functionality here as this file will be removed *
-// ***********************************************************************
-
-export class BitbucketApiReaderProcessor implements LocationProcessor {
- private username: string;
- private password: string;
-
- constructor(config: Config) {
- this.username =
- config.getOptionalString('catalog.processors.bitbucketApi.username') ??
- '';
- this.password =
- config.getOptionalString('catalog.processors.bitbucketApi.appPassword') ??
- '';
- }
-
- getRequestOptions(): RequestInit {
- const headers: HeadersInit = {};
-
- if (this.username !== '' && this.password !== '') {
- headers.Authorization = `Basic ${Buffer.from(
- `${this.username}:${this.password}`,
- 'utf8',
- ).toString('base64')}`;
- }
-
- const requestOptions: RequestInit = {
- headers,
- };
-
- return requestOptions;
- }
-
- async readLocation(
- location: LocationSpec,
- optional: boolean,
- emit: LocationProcessorEmit,
- ): Promise {
- if (location.type !== 'bitbucket/api') {
- return false;
- }
-
- try {
- const url = this.buildRawUrl(location.target);
-
- const response = await fetch(url.toString(), this.getRequestOptions());
-
- if (response.ok) {
- const data = await response.buffer();
- emit(result.data(location, data));
- } else {
- const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
- if (response.status === 404) {
- if (!optional) {
- emit(result.notFoundError(location, message));
- }
- } else {
- emit(result.generalError(location, message));
- }
- }
- } catch (e) {
- const message = `Unable to read ${location.type} ${location.target}, ${e}`;
- emit(result.generalError(location, message));
- }
- return true;
- }
-
- // Converts
- // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml
- // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml
-
- buildRawUrl(target: string): URL {
- try {
- const url = new URL(target);
-
- const [
- empty,
- userOrOrg,
- repoName,
- srcKeyword,
- ref,
- ...restOfPath
- ] = url.pathname.split('/');
-
- if (
- url.hostname !== 'bitbucket.org' ||
- empty !== '' ||
- userOrOrg === '' ||
- repoName === '' ||
- srcKeyword !== 'src'
- ) {
- throw new Error('Wrong Bitbucket URL or Invalid file path');
- }
-
- // transform to api
- url.pathname = [
- empty,
- '2.0',
- 'repositories',
- userOrOrg,
- repoName,
- 'src',
- ref,
- ...restOfPath,
- ].join('/');
- url.hostname = 'api.bitbucket.org';
- url.protocol = 'https';
-
- return url;
- } catch (e) {
- throw new Error(`Incorrect url: ${target}, ${e}`);
- }
- }
-}
diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
index 29f94dd0d7..e65714bcfd 100644
--- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts
@@ -17,7 +17,7 @@
import { LocationSpec } from '@backstage/catalog-model';
import { CodeOwnersEntry } from 'codeowners-utils';
import {
- buildCodeOwnerLocation,
+ buildCodeOwnerUrl,
buildUrl,
CodeOwnersProcessor,
findPrimaryCodeOwner,
@@ -27,19 +27,19 @@ import {
resolveCodeOwner,
} from './CodeOwnersProcessor';
-describe(CodeOwnersProcessor, () => {
+describe('CodeOwnersProcessor', () => {
+ const mockUrl = ({ basePath = '' } = {}): string =>
+ `https://github.com/spotify/backstage/blob/master/${basePath}catalog-info.yaml`;
const mockLocation = ({
basePath = '',
type = 'github',
} = {}): LocationSpec => ({
type,
- target: `https://github.com/spotify/backstage/blob/master/${basePath}catalog-info.yaml`,
+ target: mockUrl({ basePath }),
});
- const mockReadLocation = (basePath = '') => ({
- type: 'github',
- target: `https://github.com/spotify/backstage/blob/master/${basePath}CODEOWNERS`,
- });
+ const mockReadUrl = (basePath = '') =>
+ `https://github.com/spotify/backstage/blob/master/${basePath}CODEOWNERS`;
const mockGitUri = (codeOwnersPath: string = '') => {
return {
@@ -79,7 +79,7 @@ describe(CodeOwnersProcessor, () => {
return data;
};
- describe(buildUrl, () => {
+ describe('buildUrl', () => {
it.each([['azure.com'], ['dev.azure.com']])(
'should throw not implemented error',
source => {
@@ -99,37 +99,34 @@ describe(CodeOwnersProcessor, () => {
});
});
- describe(buildCodeOwnerLocation, () => {
- it('should builds a location spec to the codeowners', () => {
- expect(
- buildCodeOwnerLocation(mockLocation(), '/docs/CODEOWNERS'),
- ).toEqual({
- type: 'github',
- target:
- 'https://github.com/spotify/backstage/blob/master/docs/CODEOWNERS',
- });
+ describe('buildCodeOwnerUrl', () => {
+ it('should build a location spec to the codeowners', () => {
+ expect(buildCodeOwnerUrl(mockUrl(), '/docs/CODEOWNERS')).toEqual(
+ 'https://github.com/spotify/backstage/blob/master/docs/CODEOWNERS',
+ );
});
it('should handle nested paths from original location spec', () => {
expect(
- buildCodeOwnerLocation(
- mockLocation({ basePath: 'packages/foo/' }),
+ buildCodeOwnerUrl(
+ mockUrl({ basePath: 'packages/foo/' }),
'/CODEOWNERS',
),
- ).toEqual({
- type: 'github',
- target: 'https://github.com/spotify/backstage/blob/master/CODEOWNERS',
- });
+ ).toEqual('https://github.com/spotify/backstage/blob/master/CODEOWNERS');
});
});
- describe(parseCodeOwners, () => {
+ describe('parseCodeOwners', () => {
it('should parse the codeowners file', () => {
expect(parseCodeOwners(mockCodeOwnersText())).toEqual(mockCodeOwners());
});
});
- describe(normalizeCodeOwner, () => {
+ describe('normalizeCodeOwner', () => {
+ it('should remove the @ symbol', () => {
+ expect(normalizeCodeOwner('@yoda')).toBe('yoda');
+ });
+
it('should remove org from org/team format', () => {
expect(normalizeCodeOwner('@acme/foo')).toBe('foo');
});
@@ -146,27 +143,29 @@ describe(CodeOwnersProcessor, () => {
);
});
- describe(findPrimaryCodeOwner, () => {
+ describe('findPrimaryCodeOwner', () => {
it('should return the primary owner', () => {
expect(findPrimaryCodeOwner(mockCodeOwners())).toBe('backstage-core');
});
});
- describe(findRawCodeOwners, () => {
+ describe('findRawCodeOwners', () => {
it('should return found codeowner', async () => {
const ownersText = mockCodeOwnersText();
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: ownersText }));
- const result = await findRawCodeOwners(mockLocation(), read);
+ const reader = { read };
+ const result = await findRawCodeOwners(mockLocation(), reader);
expect(result).toEqual(ownersText);
});
it('should raise error when no codeowner', async () => {
const read = jest.fn().mockRejectedValue(mockReadResult());
+ const reader = { read };
await expect(
- findRawCodeOwners(mockLocation(), read),
+ findRawCodeOwners(mockLocation(), reader),
).rejects.toBeInstanceOf(Error);
});
@@ -177,23 +176,26 @@ describe(CodeOwnersProcessor, () => {
.mockImplementationOnce(() => mockReadResult({ error: 'foo' }))
.mockImplementationOnce(() => mockReadResult({ error: 'bar' }))
.mockResolvedValue(mockReadResult({ data: ownersText }));
+ const reader = { read };
- const result = await findRawCodeOwners(mockLocation(), read);
+ const result = await findRawCodeOwners(mockLocation(), reader);
expect(read.mock.calls.length).toBe(3);
- expect(read.mock.calls[0]).toEqual([mockReadLocation('.github/')]);
- expect(read.mock.calls[1]).toEqual([mockReadLocation('')]);
- expect(read.mock.calls[2]).toEqual([mockReadLocation('docs/')]);
+ expect(read.mock.calls[0]).toEqual([mockReadUrl('.github/')]);
+ expect(read.mock.calls[1]).toEqual([mockReadUrl('')]);
+ expect(read.mock.calls[2]).toEqual([mockReadUrl('docs/')]);
expect(result).toEqual(ownersText);
});
});
- describe(resolveCodeOwner, () => {
+ describe('resolveCodeOwner', () => {
it('should return found codeowner', async () => {
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
- const owner = await resolveCodeOwner(mockLocation(), read);
+ const reader = { read };
+
+ const owner = await resolveCodeOwner(mockLocation(), reader);
expect(owner).toBe('backstage-core');
});
@@ -201,20 +203,22 @@ describe(CodeOwnersProcessor, () => {
const read = jest
.fn()
.mockImplementation(() => mockReadResult({ error: 'error: foo' }));
+ const reader = { read };
await expect(
- resolveCodeOwner(mockLocation(), read),
+ resolveCodeOwner(mockLocation(), reader),
).rejects.toBeInstanceOf(Error);
});
});
- describe(CodeOwnersProcessor, () => {
+ describe('CodeOwnersProcessor', () => {
const setupTest = ({ kind = 'Component', spec = {} } = {}) => {
const entity = { kind, spec };
- const processor = new CodeOwnersProcessor();
const read = jest
.fn()
.mockResolvedValue(mockReadResult({ data: mockCodeOwnersText() }));
+ const reader = { read };
+ const processor = new CodeOwnersProcessor({ reader });
return { entity, processor, read };
};
@@ -224,11 +228,9 @@ describe(CodeOwnersProcessor, () => {
spec: { owner: '@acme/foo-team' },
});
- const result = await processor.processEntity(
+ const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
- null as any,
- null as any,
);
expect(result).toEqual(entity);
@@ -237,11 +239,9 @@ describe(CodeOwnersProcessor, () => {
it('should ignore url locations', async () => {
const { entity, processor } = setupTest();
- const result = await processor.processEntity(
+ const result = await processor.preProcessEntity(
entity as any,
mockLocation({ type: 'url' }),
- null as any,
- null as any,
);
expect(result).toEqual(entity);
@@ -250,24 +250,20 @@ describe(CodeOwnersProcessor, () => {
it('should ignore invalid kinds', async () => {
const { entity, processor } = setupTest({ kind: 'Group' });
- const result = await processor.processEntity(
+ const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
- null as any,
- null as any,
);
expect(result).toEqual(entity);
});
it('should set owner from codeowner', async () => {
- const { entity, processor, read } = setupTest();
+ const { entity, processor } = setupTest();
- const result = await processor.processEntity(
+ const result = await processor.preProcessEntity(
entity as any,
mockLocation(),
- null as any,
- read,
);
expect(result).toEqual({
diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts
index c875bdc799..f186a9425c 100644
--- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts
+++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.ts
@@ -14,19 +14,15 @@
* limitations under the License.
*/
+import { UrlReader } from '@backstage/backend-common';
import { Entity, LocationSpec } from '@backstage/catalog-model';
-import {
- LocationProcessor,
- LocationProcessorEmit,
- LocationProcessorRead,
-} from './types';
import * as codeowners from 'codeowners-utils';
import { CodeOwnersEntry } from 'codeowners-utils';
-import parseGitUri from 'git-url-parse';
-import { filter, head, get, pipe, reverse } from 'lodash/fp';
-
// NOTE: This can be removed when ES2021 is implemented
import 'core-js/features/promise';
+import parseGitUri from 'git-url-parse';
+import { filter, get, head, pipe, reverse } from 'lodash/fp';
+import { CatalogProcessor } from './types';
const ALLOWED_LOCATION_TYPES = [
'azure/api',
@@ -37,12 +33,16 @@ const ALLOWED_LOCATION_TYPES = [
'gitlab/api',
];
-export class CodeOwnersProcessor implements LocationProcessor {
- async processEntity(
+type Options = {
+ reader: UrlReader;
+};
+
+export class CodeOwnersProcessor implements CatalogProcessor {
+ constructor(private readonly options: Options) {}
+
+ async preProcessEntity(
entity: Entity,
location: LocationSpec,
- _emit: LocationProcessorEmit,
- read: LocationProcessorRead,
): Promise {
// Only continue if the owner is not set
if (
@@ -54,7 +54,7 @@ export class CodeOwnersProcessor implements LocationProcessor {
return entity;
}
- const owner = await resolveCodeOwner(location, read);
+ const owner = await resolveCodeOwner(location, this.options.reader);
return {
...entity,
@@ -65,9 +65,9 @@ export class CodeOwnersProcessor implements LocationProcessor {
export async function resolveCodeOwner(
location: LocationSpec,
- read: LocationProcessorRead,
+ reader: UrlReader,
): Promise