paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 47 Packages Projects Releases Wiki Activity

Merge branch 'master' into ndudnik/edit-yaml

Browse Source
This commit is contained in:
Nikita Dudnik
2020-06-03 21:45:58 +02:00
committed by GitHub
parent 6d61fba0c5 6ded51f598
commit df3cb72f6b
59 changed files with 1239 additions and 736 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/auth-backend/README.md
+8
View File
@@ -19,6 +19,14 @@ read -r AUTH_GOOGLE_CLIENT_SECRET
export AUTH_GOOGLE_CLIENT_SECRET
run `yarn start` in packages/backend folder
### SAML
To try out SAML, you can use the mock identity provider:
```bash
./scripts/start-saml-idp.sh
```
## Links
- (The Backstage homepage)[https://backstage.io]
plugins/auth-backend/package.json
+4
View File
@@ -20,6 +20,7 @@
"@types/passport": "^1.0.3",
"@types/passport-github2": "^1.2.4",
"@types/passport-google-oauth20": "^2.0.3",
"body-parser": "^1.19.0",
"compression": "^1.7.4",
"cookie-parser": "^1.4.5",
"cors": "^2.8.5",
@@ -31,11 +32,14 @@
"passport": "^0.4.1",
"passport-github2": "^0.1.12",
"passport-google-oauth20": "^2.0.0",
"passport-saml": "^1.3.3",
"winston": "^3.2.1",
"yn": "^4.0.0"
},
"devDependencies": {
"@backstage/cli": "^0.1.1-alpha.6",
"@types/body-parser": "^1.19.0",
"@types/passport-saml": "^1.1.2",
"jest-fetch-mock": "^3.0.3",
"tsc-watch": "^4.2.3"
},
plugins/auth-backend/scripts/.gitignore
+1
View File
@@ -0,0 +1 @@
*.pem
plugins/auth-backend/scripts/start-saml-idp.sh
Executable
+21
View File
@@ -0,0 +1,21 @@
#!/bin/bash
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
cd "$DIR"
if [[ ! -f idp-public-cert.pem ]]; then
echo "Generating new SAML Certificates"
openssl req \
-x509 \
-newkey rsa:1024 \
-days 3650 \
-nodes \
-subj '/CN=localhost' \
-keyout "idp-private-key.pem" \
-out "idp-public-cert.pem"
fi
echo "Downloading and starting SAML-IdP"
export NPM_CONFIG_REGISTRY=https://registry.npmjs.org
exec npx saml-idp --acsUrl "http://localhost:7000/auth/saml/handler/frame" --audience "http://localhost:7000" --port 7001
plugins/auth-backend/src/providers/PassportStrategyHelper.ts
+1 -1
View File
@@ -55,7 +55,7 @@ export const executeFrameHandlerStrategy = async (
reject(new Error('Unexpected redirect'));
};
strategy.authenticate(req);
strategy.authenticate(req, {});
});
};
plugins/auth-backend/src/providers/config.ts
+8
View File
@@ -32,4 +32,12 @@ export const providers = [
},
disableRefresh: true,
},
{
provider: 'saml',
options: {
path: '/auth/saml/handler/frame',
entryPoint: 'http://localhost:7001/',
issuer: 'passport-saml',
},
},
];
plugins/auth-backend/src/providers/factories.ts
+30 -30
View File
@@ -14,37 +14,37 @@
* limitations under the License.
*/
import {
AuthProviderFactories,
AuthProviderRouteHandlers,
AuthProviderConfig,
} from './types';
import { GoogleAuthProvider } from './google';
import { GithubAuthProvider } from './github';
import { OAuthProvider } from './OAuthProvider';
import Router from 'express-promise-router';
import { createGithubProvider } from './github';
import { createGoogleProvider } from './google';
import { createSamlProvider } from './saml';
import { AuthProviderFactory, AuthProviderConfig } from './types';
export class ProviderFactories {
private static readonly providerFactories: AuthProviderFactories = {
google: GoogleAuthProvider,
github: GithubAuthProvider,
};
const factories: { [providerId: string]: AuthProviderFactory } = {
google: createGoogleProvider,
github: createGithubProvider,
saml: createSamlProvider,
};
public static getProviderFactory(
config: AuthProviderConfig,
): AuthProviderRouteHandlers {
const providerId = config.provider;
const ProviderImpl = ProviderFactories.providerFactories[providerId];
if (!ProviderImpl) {
throw Error(
`Provider Implementation missing for : ${providerId} auth provider`,
);
}
const providerInstance = new ProviderImpl(config);
const oauthProvider = new OAuthProvider(
providerInstance,
providerId,
config.disableRefresh,
);
return oauthProvider;
export function createAuthProvider(providerId: string, config: any) {
const factory = factories[providerId];
if (!factory) {
throw Error(`No auth provider available for '${providerId}'`);
}
return factory(config);
}
export const createAuthProviderRouter = (config: AuthProviderConfig) => {
const providerId = config.provider;
const provider = createAuthProvider(providerId, config);
const router = Router();
router.get('/start', provider.start.bind(provider));
router.get('/handler/frame', provider.frameHandler.bind(provider));
router.post('/handler/frame', provider.frameHandler.bind(provider));
router.get('/logout', provider.logout.bind(provider));
if (provider.refresh) {
router.get('/refresh', provider.refresh.bind(provider));
}
return router;
};
plugins/auth-backend/src/providers/github/index.ts
+1 -1
View File
@@ -14,4 +14,4 @@
* limitations under the License.
*/
export { GithubAuthProvider } from './provider';
export { createGithubProvider } from './provider';
plugins/auth-backend/src/providers/github/provider.ts
+7
View File
@@ -27,6 +27,7 @@ import {
AuthInfoBase,
AuthInfoPrivate,
} from '../types';
import { OAuthProvider } from '../OAuthProvider';
export class GithubAuthProvider implements OAuthProviderHandlers {
private readonly providerConfig: AuthProviderConfig;
@@ -57,3 +58,9 @@ export class GithubAuthProvider implements OAuthProviderHandlers {
return await executeFrameHandlerStrategy(req, this._strategy);
}
}
export function createGithubProvider(config: AuthProviderConfig) {
const provider = new GithubAuthProvider(config);
const oauthProvider = new OAuthProvider(provider, config.provider, true);
return oauthProvider;
}
plugins/auth-backend/src/providers/google/index.ts
+1 -1
View File
@@ -14,4 +14,4 @@
* limitations under the License.
*/
export { GoogleAuthProvider } from './provider';
export { createGoogleProvider } from './provider';
plugins/auth-backend/src/providers/google/provider.ts
+7
View File
@@ -28,6 +28,7 @@ import {
RedirectInfo,
AuthProviderConfig,
} from '../types';
import { OAuthProvider } from '../OAuthProvider';
export class GoogleAuthProvider implements OAuthProviderHandlers {
private readonly providerConfig: AuthProviderConfig;
@@ -87,3 +88,9 @@ export class GoogleAuthProvider implements OAuthProviderHandlers {
};
}
}
export function createGoogleProvider(config: AuthProviderConfig) {
const provider = new GoogleAuthProvider(config);
const oauthProvider = new OAuthProvider(provider, config.provider);
return oauthProvider;
}
plugins/auth-backend/src/providers/index.ts
+1 -21
View File
@@ -14,24 +14,4 @@
* limitations under the License.
*/
import Router from 'express-promise-router';
import { AuthProviderRouteHandlers, AuthProviderConfig } from './types';
import { ProviderFactories } from './factories';
export const defaultRouter = (provider: AuthProviderRouteHandlers) => {
const router = Router();
router.get('/start', provider.start.bind(provider));
router.get('/handler/frame', provider.frameHandler.bind(provider));
router.get('/logout', provider.logout.bind(provider));
if (provider.refresh) {
router.get('/refresh', provider.refresh.bind(provider));
}
return router;
};
export const makeProvider = (config: AuthProviderConfig) => {
const providerId = config.provider;
const oauthProvider = ProviderFactories.getProviderFactory(config);
const providerRouter = defaultRouter(oauthProvider);
return { providerId, providerRouter };
};
export { createAuthProviderRouter } from './factories';
plugins/auth-backend/src/providers/index.test.ts → plugins/auth-backend/src/providers/saml/index.ts
+1 -7
View File
@@ -14,10 +14,4 @@
* limitations under the License.
*/
import { defaultRouter } from '.';
describe('test', () => {
it('unbreaks the test runner', () => {
expect(defaultRouter).toBeDefined();
});
});
export { createSamlProvider } from './provider';
plugins/auth-backend/src/providers/saml/provider.ts
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import express from 'express';
import { Strategy as SamlStrategy } from 'passport-saml';
import {
executeFrameHandlerStrategy,
executeRedirectStrategy,
} from '../PassportStrategyHelper';
import { AuthProviderConfig, AuthProviderRouteHandlers } from '../types';
import { postMessageResponse } from '../OAuthProvider';
export class SamlAuthProvider implements AuthProviderRouteHandlers {
private readonly strategy: SamlStrategy;
constructor(providerConfig: AuthProviderConfig) {
this.strategy = new SamlStrategy(
{ ...providerConfig.options },
(profile: any, done: any) => {
// TODO: There's plenty more validation and profile handling to do here,
// this provider is currently only intended to validate the provider pattern
// for non-oauth auth flows.
// TODO: This flow doesn't issue an identity token that can be used to validate
// the identity of the user in other backends, which we need in some form.
done(undefined, {
email: profile.email,
firstName: profile.firstName,
lastName: profile.lastName,
displayName: profile.displayName,
});
},
);
}
async start(req: express.Request, res: express.Response): Promise<any> {
const { url } = await executeRedirectStrategy(req, this.strategy, {});
res.redirect(url);
}
async frameHandler(
req: express.Request,
res: express.Response,
): Promise<any> {
try {
const { user } = await executeFrameHandlerStrategy(req, this.strategy);
return postMessageResponse(res, {
type: 'auth-result',
payload: user,
});
} catch (error) {
return postMessageResponse(res, {
type: 'auth-result',
error: {
name: error.name,
message: error.message,
},
});
}
}
async logout(_req: express.Request, res: express.Response): Promise<any> {
res.send('noop');
}
}
export function createSamlProvider(config: AuthProviderConfig) {
return new SamlAuthProvider(config);
}
plugins/auth-backend/src/providers/types.ts
+3 -7
View File
@@ -37,13 +37,9 @@ export interface AuthProviderRouteHandlers {
logout(req: express.Request, res: express.Response): Promise<any>;
}
export type AuthProviderFactories = {
[key: string]: AuthProviderFactory;
};
export type AuthProviderFactory = {
new (providerConfig: any): OAuthProviderHandlers;
};
export type AuthProviderFactory = (
config: AuthProviderConfig,
) => AuthProviderRouteHandlers;
export type AuthInfoBase = {
accessToken: string;
plugins/auth-backend/src/service/router.ts
+8 -4
View File
@@ -17,9 +17,10 @@
import express from 'express';
import Router from 'express-promise-router';
import cookieParser from 'cookie-parser';
import bodyParser from 'body-parser';
import { Logger } from 'winston';
import { providers } from './../providers/config';
import { makeProvider } from '../providers';
import { createAuthProviderRouter } from '../providers';
export interface RouterOptions {
logger: Logger;
@@ -32,12 +33,15 @@ export async function createRouter(
const logger = options.logger.child({ plugin: 'auth' });
router.use(cookieParser());
router.use(bodyParser.urlencoded({ extended: false }));
router.use(bodyParser.json());
// configure all the providers
for (const providerConfig of providers) {
const { providerId, providerRouter } = makeProvider(providerConfig);
logger.info(`Configuring provider, ${providerId}`);
router.use(`/${providerId}`, providerRouter);
const { provider } = providerConfig;
const providerRouter = createAuthProviderRouter(providerConfig);
logger.info(`Configuring provider, ${provider}`);
router.use(`/${provider}`, providerRouter);
}
return router;
plugins/catalog-backend/package.json
+1 -1
View File
@@ -6,7 +6,7 @@
"license": "Apache-2.0",
"private": true,
"scripts": {
"start": "tsc-watch --onFirstSuccess \"cross-env NODE_ENV=development nodemon -r esm dist/run.js\"",
"start": "backstage-cli watch-deps --build -- tsc-watch --onFirstSuccess \\\"cross-env NODE_ENV=development nodemon -r esm dist/run.js\\\"",
"build": "tsc",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
plugins/catalog-backend/src/catalog/DatabaseEntitiesCatalog.ts
+1 -1
View File
@@ -38,8 +38,8 @@ export class DatabaseEntitiesCatalog implements EntitiesCatalog {
async entityByName(
kind: string,
name: string,
namespace: string | undefined,
name: string,
): Promise<Entity | undefined> {
return await this.database.transaction(tx =>
this.entityByNameInternal(tx, kind, name, namespace),
plugins/catalog-backend/src/catalog/DatabaseLocationsCatalog.ts
+28 -1
View File
@@ -16,7 +16,10 @@
import { Location } from '@backstage/catalog-model';
import type { Database } from '../database';
import { DatabaseLocationUpdateLogEvent } from '../database/types';
import {
DatabaseLocationUpdateLogEvent,
DatabaseLocationUpdateLogStatus,
} from '../database/types';
import { LocationResponse, LocationsCatalog } from './types';
export class DatabaseLocationsCatalog implements LocationsCatalog {
@@ -63,4 +66,28 @@ export class DatabaseLocationsCatalog implements LocationsCatalog {
data,
};
}
async logUpdateSuccess(
locationId: string,
entityName?: string,
): Promise<void> {
await this.database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.SUCCESS,
entityName,
);
}
async logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void> {
await this.database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.FAIL,
entityName,
error?.message,
);
}
}
plugins/catalog-backend/src/catalog/types.ts
+6
View File
@@ -62,4 +62,10 @@ export type LocationsCatalog = {
locations(): Promise<LocationResponse[]>;
location(id: string): Promise<LocationResponse>;
locationHistory(id: string): Promise<LocationUpdateLogEvent[]>;
logUpdateSuccess(locationId: string, entityName?: string): Promise<void>;
logUpdateFailure(
locationId: string,
error?: Error,
entityName?: string,
): Promise<void>;
};
plugins/catalog-backend/src/database/DatabaseManager.test.ts
-248
View File
@@ -1,248 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import type { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import type { IngestionModel } from '../ingestion/types';
import { DatabaseManager } from './DatabaseManager';
import { DatabaseLocationUpdateLogStatus } from './types';
import type {
Database,
DbLocationsRow,
DbLocationsRowWithStatus,
} from './types';
describe('DatabaseManager', () => {
describe('refreshLocations', () => {
it('works with no locations added', async () => {
const db = ({
locations: jest.fn().mockResolvedValue([]),
} as unknown) as Database;
const reader: IngestionModel = {
readLocation: jest.fn(),
};
const policy: EntityPolicy = {
enforce: jest.fn(),
};
await expect(
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.readLocation).not.toHaveBeenCalled();
expect(policy.enforce).not.toHaveBeenCalled();
});
it('can update a single location', async () => {
const location: DbLocationsRowWithStatus = {
id: '123',
type: 'some',
target: 'thing',
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const tx = (undefined as unknown) as Knex.Transaction<any, any>;
const db = ({
transaction: jest.fn(f => f(tx)),
entity: jest.fn(() => Promise.resolve(undefined)),
addEntity: jest.fn(),
locations: jest.fn(() => Promise.resolve([location])),
addLocationUpdateLogEvent: jest.fn(),
} as Partial<Database>) as Database;
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(reader.readLocation).toHaveBeenCalledTimes(1);
expect(reader.readLocation).toHaveBeenNthCalledWith(1, 'some', 'thing');
expect(db.addEntity).toHaveBeenCalledTimes(1);
expect(db.addEntity).toHaveBeenNthCalledWith(1, undefined, {
locationId: '123',
entity: expect.objectContaining({
metadata: expect.objectContaining({ name: 'c1' }),
}),
});
});
it('logs successful updates', async () => {
const tx = (undefined as unknown) as Knex.Transaction<any, any>;
const db = ({
transaction: jest.fn(f => f(tx)),
addEntity: jest.fn(),
entity: jest.fn(() => Promise.resolve(undefined)),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const policy: EntityPolicy = {
enforce: jest.fn(() => Promise.resolve(desc)),
};
await expect(
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
'c1',
);
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
2,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
undefined,
);
});
it('logs unsuccessful updates when parser fails', async () => {
const tx = (undefined as unknown) as Knex.Transaction<any, any>;
const db = ({
transaction: jest.fn(f => f(tx)),
addEntity: jest.fn(),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.resolve([{ type: 'data', data: desc }]),
),
};
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.FAIL,
'c1',
'parser error message',
);
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
2,
'123',
DatabaseLocationUpdateLogStatus.SUCCESS,
undefined,
);
});
it('logs unsuccessful updates when reader fails', async () => {
const tx = (undefined as unknown) as Knex.Transaction<any, any>;
const db = ({
transaction: jest.fn(f => f(tx)),
addEntity: jest.fn(),
locations: jest.fn(() =>
Promise.resolve([
{
id: '123',
type: 'some',
target: 'thing',
} as DbLocationsRow,
]),
),
addLocationUpdateLogEvent: jest.fn(),
} as unknown) as Database;
const reader: IngestionModel = {
readLocation: jest.fn(() =>
Promise.reject([{ type: 'error', error: new Error('test message') }]),
),
};
const policy: EntityPolicy = {
enforce: jest.fn(() =>
Promise.reject(new Error('parser error message')),
),
};
await expect(
DatabaseManager.refreshLocations(db, reader, policy, getVoidLogger()),
).resolves.toBeUndefined();
expect(db.addLocationUpdateLogEvent).toHaveBeenNthCalledWith(
1,
'123',
DatabaseLocationUpdateLogStatus.FAIL,
undefined,
undefined,
);
});
});
});
plugins/catalog-backend/src/database/DatabaseManager.ts
+1 -186
View File
@@ -14,18 +14,11 @@
* limitations under the License.
*/
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import Knex from 'knex';
import lodash from 'lodash';
import path from 'path';
import { Logger } from 'winston';
import { IngestionModel } from '../ingestion/types';
import { CommonDatabase } from './CommonDatabase';
import {
DatabaseLocationUpdateLogStatus,
Database,
DbEntityRequest,
} from './types';
import { Database } from './types';
export class DatabaseManager {
public static async createDatabase(
@@ -57,182 +50,4 @@ export class DatabaseManager {
});
return new CommonDatabase(database, logger);
}
private static async logUpdateSuccess(
database: Database,
locationId: string,
entityName?: string,
) {
return database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.SUCCESS,
entityName,
);
}
private static async logUpdateFailure(
database: Database,
locationId: string,
error?: Error,
entityName?: string,
) {
return database.addLocationUpdateLogEvent(
locationId,
DatabaseLocationUpdateLogStatus.FAIL,
entityName,
error?.message,
);
}
public static async refreshLocations(
database: Database,
ingestionModel: IngestionModel,
entityPolicy: EntityPolicy,
logger: Logger,
): Promise<void> {
const locations = await database.locations();
for (const location of locations) {
try {
logger.debug(
`Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
);
const readerOutput = await ingestionModel.readLocation(
location.type,
location.target,
);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
logger.info(readerItem.error);
continue;
}
try {
const entity = await entityPolicy.enforce(readerItem.data);
await DatabaseManager.refreshSingleEntity(
database,
location.id,
entity,
logger,
);
await DatabaseManager.logUpdateSuccess(
database,
location.id,
entity.metadata.name,
);
} catch (error) {
await DatabaseManager.logUpdateFailure(
database,
location.id,
error,
readerItem.data.metadata.name,
);
}
}
await DatabaseManager.logUpdateSuccess(
database,
location.id,
undefined,
);
} catch (error) {
logger.debug(
`Failed to refresh location id="${location.id}", ${error}`,
);
await DatabaseManager.logUpdateFailure(database, location.id, error);
}
}
}
private static async refreshSingleEntity(
database: Database,
locationId: string,
entity: Entity,
logger: Logger,
): Promise<void> {
const { kind } = entity;
const { name, namespace } = entity.metadata || {};
if (!name) {
throw new Error('Entities without names are not yet supported');
}
const request: DbEntityRequest = {
locationId: locationId,
entity: entity,
};
logger.debug(
`Read entity kind="${kind}" name="${name}" namespace="${namespace}"`,
);
await database.transaction(async tx => {
const previous = await database.entity(tx, kind, name, namespace);
if (!previous) {
logger.debug(`No such entity found, adding`);
await database.addEntity(tx, request);
} else if (
!DatabaseManager.entitiesAreEqual(previous.entity, request.entity)
) {
logger.debug(`Different from existing entity, updating`);
await database.updateEntity(tx, request);
} else {
logger.debug(`Equal to existing entity, skipping update`);
}
});
}
private static entitiesAreEqual(previous: Entity, next: Entity) {
if (
previous.apiVersion !== next.apiVersion ||
previous.kind !== next.kind ||
!lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined
) {
return false;
}
// Since the next annotations get merged into the previous, extract only
// the overlapping keys and check if their values match.
if (next.metadata.annotations) {
if (!previous.metadata.annotations) {
return false;
}
if (
!lodash.isEqual(
next.metadata.annotations,
lodash.pick(
previous.metadata.annotations,
Object.keys(next.metadata.annotations),
),
)
) {
return false;
}
}
const e1 = lodash.cloneDeep(previous);
const e2 = lodash.cloneDeep(next);
if (!e1.metadata.labels) {
e1.metadata.labels = {};
}
if (!e2.metadata.labels) {
e2.metadata.labels = {};
}
// Remove generated fields
delete e1.metadata.uid;
delete e1.metadata.etag;
delete e1.metadata.generation;
delete e2.metadata.uid;
delete e2.metadata.etag;
delete e2.metadata.generation;
// Remove already compared things
delete e1.metadata.annotations;
delete e1.spec;
delete e2.metadata.annotations;
delete e2.spec;
return lodash.isEqual(e1, e2);
}
}
plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
+151 -2
View File
@@ -14,10 +14,13 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, Location } from '@backstage/catalog-model';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { IngestionModel } from './types';
import { LocationUpdateStatus } from '../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../database/types';
import { HigherOrderOperations } from './HigherOrderOperations';
import { Entity } from '@backstage/catalog-model';
import { IngestionModel } from './types';
describe('HigherOrderOperations', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
@@ -39,6 +42,8 @@ describe('HigherOrderOperations', () => {
locations: jest.fn(),
location: jest.fn(),
locationHistory: jest.fn(),
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
ingestionModel = {
readLocation: jest.fn(),
@@ -47,6 +52,7 @@ describe('HigherOrderOperations', () => {
entitiesCatalog,
locationsCatalog,
ingestionModel,
getVoidLogger(),
);
});
@@ -140,4 +146,147 @@ describe('HigherOrderOperations', () => {
expect(locationsCatalog.addLocation).not.toBeCalled();
});
});
describe('refreshLocations', () => {
it('works with no locations added', async () => {
locationsCatalog.locations.mockResolvedValue([]);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).not.toHaveBeenCalled();
expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled();
});
it('can update a single location where a matching entity did not exist', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockResolvedValue([
{ type: 'data', data: desc },
]);
entitiesCatalog.entityByName.mockResolvedValue(undefined);
entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).toHaveBeenNthCalledWith(
1,
'some',
'thing',
);
expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByName).toHaveBeenNthCalledWith(
1,
'Component',
undefined,
'c1',
);
expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
metadata: expect.objectContaining({ name: 'c1' }),
}),
'123',
);
});
it('logs successful updates', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
const desc: Entity = {
apiVersion: 'backstage.io/v1beta1',
kind: 'Component',
metadata: { name: 'c1' },
spec: { type: 'service' },
};
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockResolvedValue([
{ type: 'data', data: desc },
]);
entitiesCatalog.entityByName.mockResolvedValue(undefined);
entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledTimes(2);
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith(
'123',
undefined,
);
expect(locationsCatalog.logUpdateSuccess).toHaveBeenCalledWith(
'123',
'c1',
);
});
it('logs unsuccessful updates when reader fails', async () => {
const locationStatus: LocationUpdateStatus = {
message: '',
status: DatabaseLocationUpdateLogStatus.SUCCESS,
timestamp: new Date(314159265).toISOString(),
};
const location: Location = {
id: '123',
type: 'some',
target: 'thing',
};
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockRejectedValue(
new Error('reader error message'),
);
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled();
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith(
'123',
expect.objectContaining({ message: 'reader error message' }),
);
});
});
});
plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
+155
View File
@@ -16,10 +16,12 @@
import { InputError } from '@backstage/backend-common';
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { IngestionModel } from '../ingestion';
import { AddLocationResult, HigherOrderOperation } from './types';
import { Logger } from 'winston';
const LOCATION_ANNOTATION = 'backstage.io/managed-by-location';
@@ -34,15 +36,18 @@ export class HigherOrderOperations implements HigherOrderOperation {
private readonly entitiesCatalog: EntitiesCatalog;
private readonly locationsCatalog: LocationsCatalog;
private readonly ingestionModel: IngestionModel;
private readonly logger: Logger;
constructor(
entitiesCatalog: EntitiesCatalog,
locationsCatalog: LocationsCatalog,
ingestionModel: IngestionModel,
logger: Logger,
) {
this.entitiesCatalog = entitiesCatalog;
this.locationsCatalog = locationsCatalog;
this.ingestionModel = ingestionModel;
this.logger = logger;
}
/**
@@ -111,4 +116,154 @@ export class HigherOrderOperations implements HigherOrderOperation {
return { location, entities: outputEntities };
}
/**
* Goes through all registered locations, and performs a refresh of each one.
*
* Entities are read from their respective sources, are parsed and validated
* according to the entity policy, and get inserted or updated in the catalog.
* Entities that have disappeared from their location are left orphaned,
* without changes.
*/
async refreshAllLocations(): Promise<void> {
const startTimestamp = new Date().valueOf();
this.logger.info('Beginning locations refresh');
const locations = await this.locationsCatalog.locations();
this.logger.info(`Visiting ${locations.length} locations`);
for (const { data: location } of locations) {
this.logger.debug(
`Refreshing location id="${location.id}" type="${location.type}" target="${location.target}"`,
);
try {
await this.refreshSingleLocation(location);
await this.locationsCatalog.logUpdateSuccess(location.id, undefined);
} catch (e) {
this.logger.debug(
`Failed to refresh location id="${location.id}" type="${location.type}" target="${location.target}", ${e}`,
);
await this.locationsCatalog.logUpdateFailure(location.id, e);
}
}
const endTimestamp = new Date().valueOf();
const duration = ((endTimestamp - startTimestamp) / 1000).toFixed(1);
this.logger.debug(`Completed locations refresh in ${duration} seconds`);
}
// Performs a full refresh of a single location
private async refreshSingleLocation(location: Location) {
const readerOutput = await this.ingestionModel.readLocation(
location.type,
location.target,
);
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
this.logger.debug(
`Failed item in location id="${location.id}" type="${location.type}" target="${location.target}", ${readerItem.error}`,
);
continue;
}
const entity = readerItem.data;
this.logger.debug(
`Read entity kind="${entity.kind}" name="${
entity.metadata.name
}" namespace="${entity.metadata.namespace || ''}"`,
);
try {
const previous = await this.entitiesCatalog.entityByName(
entity.kind,
entity.metadata.namespace,
entity.metadata.name,
);
if (!previous) {
this.logger.debug(`No such entity found, adding`);
await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
} else if (!this.entitiesAreEqual(previous, entity)) {
this.logger.debug(`Different from existing entity, updating`);
await this.entitiesCatalog.addOrUpdateEntity(entity, location.id);
} else {
this.logger.debug(`Equal to existing entity, skipping update`);
}
await this.locationsCatalog.logUpdateSuccess(
location.id,
entity.metadata.name,
);
} catch (error) {
this.logger.debug(
`Failed refresh of entity kind="${entity.kind}" name="${
entity.metadata.name
}" namespace="${entity.metadata.namespace || ''}", ${error}`,
);
await this.locationsCatalog.logUpdateFailure(
location.id,
error,
entity.metadata.name,
);
}
}
}
// Compares entities, ignoring generated and irrelevant data
private entitiesAreEqual(previous: Entity, next: Entity): boolean {
if (
previous.apiVersion !== next.apiVersion ||
previous.kind !== next.kind ||
!lodash.isEqual(previous.spec, next.spec) // Accept that {} !== undefined
) {
return false;
}
// Since the next annotations get merged into the previous, extract only
// the overlapping keys and check if their values match.
if (next.metadata.annotations) {
if (!previous.metadata.annotations) {
return false;
}
if (
!lodash.isEqual(
next.metadata.annotations,
lodash.pick(
previous.metadata.annotations,
Object.keys(next.metadata.annotations),
),
)
) {
return false;
}
}
const e1 = lodash.cloneDeep(previous);
const e2 = lodash.cloneDeep(next);
if (!e1.metadata.labels) {
e1.metadata.labels = {};
}
if (!e2.metadata.labels) {
e2.metadata.labels = {};
}
// Remove generated fields
delete e1.metadata.uid;
delete e1.metadata.etag;
delete e1.metadata.generation;
delete e2.metadata.uid;
delete e2.metadata.etag;
delete e2.metadata.generation;
// Remove already compared things
delete e1.metadata.annotations;
delete e1.spec;
delete e2.metadata.annotations;
delete e2.spec;
return lodash.isEqual(e1, e2);
}
}
plugins/catalog-backend/src/ingestion/index.ts
+1 -1
View File
@@ -18,4 +18,4 @@ export * from './descriptor';
export { HigherOrderOperations } from './HigherOrderOperations';
export { IngestionModels } from './IngestionModels';
export * from './source';
export type { IngestionModel } from './types';
export type { HigherOrderOperation, IngestionModel } from './types';
plugins/catalog-backend/src/ingestion/types.ts
+1
View File
@@ -28,4 +28,5 @@ export type IngestionModel = {
export type HigherOrderOperation = {
addLocation(spec: LocationSpec): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
};
plugins/catalog-backend/src/service/router.test.ts
+32 -12
View File
@@ -43,9 +43,12 @@ describe('createRouter', () => {
locations: jest.fn(),
location: jest.fn(),
locationHistory: jest.fn(),
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
higherOrderOperation = {
addLocation: jest.fn(),
refreshAllLocations: jest.fn(),
};
const router = await createRouter({
entitiesCatalog,
@@ -78,6 +81,7 @@ describe('createRouter', () => {
const response = await request(app).get('/entities?a=1&a=&a=3&b=4&c=');
expect(response.status).toEqual(200);
expect(entitiesCatalog.entities).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entities).toHaveBeenCalledWith([
{ key: 'a', values: ['1', null, '3'] },
{ key: 'b', values: ['4'] },
@@ -99,14 +103,19 @@ describe('createRouter', () => {
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByUid).toHaveBeenCalledWith('zzz');
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entityByUid.mockResolvedValue(undefined);
const response = await request(app).get('/entities/by-uid/zzz');
expect(entitiesCatalog.entityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByUid).toHaveBeenCalledWith('zzz');
expect(response.status).toEqual(404);
expect(response.text).toMatch(/uid/);
});
@@ -116,16 +125,18 @@ describe('createRouter', () => {
it('can fetch entity by name', async () => {
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
kind: 'k',
metadata: {
name: 'c',
namespace: 'd',
name: 'n',
namespace: 'ns',
},
};
entitiesCatalog.entityByName.mockResolvedValue(entity);
const response = await request(app).get('/entities/by-name/b/d/c');
const response = await request(app).get('/entities/by-name/k/ns/n');
expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByName).toHaveBeenCalledWith('k', 'ns', 'n');
expect(response.status).toEqual(200);
expect(response.body).toEqual(expect.objectContaining(entity));
});
@@ -133,8 +144,10 @@ describe('createRouter', () => {
it('responds with a 404 for missing entities', async () => {
entitiesCatalog.entityByName.mockResolvedValue(undefined);
const response = await request(app).get('/entities/by-name//b/d/c');
const response = await request(app).get('/entities/by-name/b/d/c');
expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByName).toHaveBeenCalledWith('b', 'd', 'c');
expect(response.status).toEqual(404);
expect(response.text).toMatch(/name/);
});
@@ -147,9 +160,9 @@ describe('createRouter', () => {
.set('Content-Type', 'application/json')
.send();
expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled();
expect(response.status).toEqual(400);
expect(response.text).toMatch(/body/);
expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled();
});
it('passes the body down', async () => {
@@ -169,13 +182,13 @@ describe('createRouter', () => {
.send(entity)
.set('Content-Type', 'application/json');
expect(response.status).toEqual(200);
expect(response.body).toEqual(entity);
expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.addOrUpdateEntity).toHaveBeenNthCalledWith(
1,
entity,
);
expect(response.status).toEqual(200);
expect(response.body).toEqual(entity);
});
});
@@ -185,8 +198,9 @@ describe('createRouter', () => {
const response = await request(app).delete('/entities/by-uid/apa');
expect(response.status).toEqual(204);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
expect(response.status).toEqual(204);
});
it('responds with a 404 for missing entities', async () => {
@@ -196,8 +210,9 @@ describe('createRouter', () => {
const response = await request(app).delete('/entities/by-uid/apa');
expect(response.status).toEqual(404);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.removeEntityByUid).toHaveBeenCalledWith('apa');
expect(response.status).toEqual(404);
});
});
@@ -227,8 +242,8 @@ describe('createRouter', () => {
const response = await request(app).post('/locations').send(spec);
expect(response.status).toEqual(400);
expect(higherOrderOperation.addLocation).not.toHaveBeenCalled();
expect(response.status).toEqual(400);
});
it('passes the body down', async () => {
@@ -244,9 +259,14 @@ describe('createRouter', () => {
const response = await request(app).post('/locations').send(spec);
expect(response.status).toEqual(201);
expect(higherOrderOperation.addLocation).toHaveBeenCalledTimes(1);
expect(higherOrderOperation.addLocation).toHaveBeenCalledWith(spec);
expect(response.status).toEqual(201);
expect(response.body).toEqual(
expect.objectContaining({
location: { id: 'a', ...spec },
}),
);
});
});
});
plugins/catalog-backend/src/service/router.ts
+1 -1
View File
@@ -69,8 +69,8 @@ export async function createRouter(
const { kind, namespace, name } = req.params;
const entity = await entitiesCatalog.entityByName(
kind,
name,
namespace,
name,
);
if (!entity) {
res
plugins/catalog-backend/src/service/standaloneApplication.ts
+2 -1
View File
@@ -25,6 +25,7 @@ import express from 'express';
import helmet from 'helmet';
import { Logger } from 'winston';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { HigherOrderOperation } from '../ingestion';
import { createRouter } from './router';
import { HigherOrderOperation } from '../ingestion/types';
@@ -43,8 +44,8 @@ export async function createStandaloneApplication(
enableCors,
entitiesCatalog,
locationsCatalog,
logger,
higherOrderOperation,
logger,
} = options;
const app = express();
plugins/graphiql/package.json
+1 -1
View File
@@ -48,7 +48,7 @@
"@testing-library/jest-dom": "^5.7.0",
"@testing-library/react": "^9.3.2",
"@testing-library/user-event": "^10.2.4",
"@types/codemirror": "^0.0.93",
"@types/codemirror": "^0.0.95",
"@types/jest": "^25.2.2",
"@types/node": "^12.0.0",
"@types/testing-library__jest-dom": "^5.0.4",
plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.componentId}}/.github/workflows/build.yml
Vendored
+1 -1
View File
@@ -19,7 +19,7 @@ jobs:
- name: get yarn cache
id: yarn-cache
run: echo "::set-output name=dir::$(yarn cache dir)"
- uses: actions/cache@v1
- uses: actions/cache@v2
with:
path: ${{ steps.yarn-cache.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}