paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

Merge pull request #8506 from backstage/authz-search-backend

Browse Source 
search: filter unauthorized results
This commit is contained in:
MT Lewis
2022-01-27 11:01:51 +00:00
committed by GitHub
parent caab696f28 af4864b462
commit c46a99b045
30 changed files with 1102 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/angry-carrots-brush.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-search-backend-node': patch
---
Track visibility permissions by document type in IndexBuilder
.changeset/angry-jeans-bow.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
Add support for permissions to the DefaultCatalogCollator.
.changeset/heavy-jars-repair.md
+25
View File
@@ -0,0 +1,25 @@
---
'@backstage/create-app': patch
---
Added three additional required properties to the search-backend `createRouter` function to support filtering search results based on permissions. To make this change to an existing app, add the required parameters to the `createRouter` call in `packages/backend/src/plugins/search.ts`:
```diff
export default async function createPlugin({
logger,
+ permissions,
discovery,
config,
tokenManager,
}: PluginEnvironment) {
/* ... */
return await createRouter({
engine: indexBuilder.getSearchEngine(),
+ types: indexBuilder.getDocumentTypes(),
+ permissions,
+ config,
logger,
});
}
```
.changeset/nasty-dodos-complain.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/search-common': patch
---
- Add optional visibilityPermission property to DocumentCollator type
- Add new DocumentTypeInfo type for housing information about the document types stored in a search engine.
.changeset/nasty-moose-trade.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/search-common': patch
---
Add optional resourceRef field to the IndexableDocument type for use when authorizing access to documents.
.changeset/nervous-walls-exercise.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/search-common': patch
---
Add optional query request options containing authorization token to SearchEngine#query.
.changeset/pink-spoons-hope.md
+25
View File
@@ -0,0 +1,25 @@
---
'@backstage/plugin-search-backend': minor
---
**BREAKING** Added three additional required properties to `createRouter` to support filtering search results based on permissions. To make this change to an existing app, add the required parameters to the `createRouter` call in `packages/backend/src/plugins/search.ts`:
```diff
export default async function createPlugin({
logger,
+ permissions,
discovery,
config,
tokenManager,
}: PluginEnvironment) {
/* ... */
return await createRouter({
engine: indexBuilder.getSearchEngine(),
+ types: indexBuilder.getDocumentTypes(),
+ permissions,
+ config,
logger,
});
}
```
.changeset/small-avocados-decide.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-techdocs-backend': patch
---
Add support for permissions to the DefaultTechDocsCollator.
packages/backend/src/plugins/search.ts
+4
View File
@@ -56,6 +56,7 @@ async function createSearchEngine({
export default async function createPlugin({
logger,
permissions,
discovery,
config,
database,
@@ -95,6 +96,9 @@ export default async function createPlugin({
return await createRouter({
engine: indexBuilder.getSearchEngine(),
types: indexBuilder.getDocumentTypes(),
permissions,
config,
logger,
});
}
packages/create-app/templates/default-app/packages/backend/src/plugins/search.ts
+4
View File
@@ -10,6 +10,7 @@ import { DefaultTechDocsCollator } from '@backstage/plugin-techdocs-backend';
export default async function createPlugin({
logger,
permissions,
discovery,
config,
tokenManager,
@@ -49,6 +50,9 @@ export default async function createPlugin({
return await createRouter({
engine: indexBuilder.getSearchEngine(),
types: indexBuilder.getDocumentTypes(),
permissions,
config,
logger,
});
}
packages/search-common/api-report.md
+23 -1
View File
@@ -4,6 +4,7 @@
```ts
import { JsonObject } from '@backstage/types';
import { Permission } from '@backstage/plugin-permission-common';
// Warning: (ae-missing-release-tag) "DocumentCollator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
@@ -12,6 +13,7 @@ export interface DocumentCollator {
// (undocumented)
execute(): Promise<IndexableDocument[]>;
readonly type: string;
readonly visibilityPermission?: Permission;
}
// Warning: (ae-missing-release-tag) "DocumentDecorator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -23,15 +25,32 @@ export interface DocumentDecorator {
readonly types?: string[];
}
// Warning: (ae-missing-release-tag) "DocumentTypeInfo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export type DocumentTypeInfo = {
visibilityPermission?: Permission;
};
// Warning: (ae-missing-release-tag) "IndexableDocument" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export interface IndexableDocument {
authorization?: {
resourceRef: string;
};
location: string;
text: string;
title: string;
}
// Warning: (ae-missing-release-tag) "QueryRequestOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type QueryRequestOptions = {
token?: string;
};
// Warning: (ae-missing-release-tag) "QueryTranslator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
@@ -42,7 +61,10 @@ export type QueryTranslator = (query: SearchQuery) => unknown;
// @public
export interface SearchEngine {
index(type: string, documents: IndexableDocument[]): Promise<void>;
query(query: SearchQuery): Promise<SearchResultSet>;
query(
query: SearchQuery,
options?: QueryRequestOptions,
): Promise<SearchResultSet>;
setTranslator(translator: QueryTranslator): void;
}
packages/search-common/package.json
+2 -1
View File
@@ -36,7 +36,8 @@
"url": "https://github.com/backstage/backstage/issues"
},
"dependencies": {
"@backstage/types": "^0.1.1"
"@backstage/types": "^0.1.1",
"@backstage/plugin-permission-common": "^0.4.0-next.0"
},
"devDependencies": {
"@backstage/cli": "^0.12.0-next.0"
packages/search-common/src/types.ts
+41 -1
View File
@@ -13,6 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Permission } from '@backstage/plugin-permission-common';
import { JsonObject } from '@backstage/types';
export interface SearchQuery {
@@ -53,8 +55,32 @@ export interface IndexableDocument {
* is clicked).
*/
location: string;
/**
* Optional authorization information to be used when determining whether this
* search result should be visible to a given user.
*/
authorization?: {
/**
* Identifier for the resource.
*/
resourceRef: string;
};
}
/**
* Information about a specific document type. Intended to be used in the
* {@link @backstage/search-backend-node#IndexBuilder} to collect information
* about the types stored in the index.
*/
export type DocumentTypeInfo = {
/**
* The {@link @backstage/plugin-permission-common#Permission} that controls
* visibility of resources associated with this collator's documents.
*/
visibilityPermission?: Permission;
};
/**
* Interface that must be implemented in order to expose new documents to
* search.
@@ -65,6 +91,13 @@ export interface DocumentCollator {
* index name by Search Engines.
*/
readonly type: string;
/**
* The {@link @backstage/plugin-permission-common#Permission} that controls
* visibility of resources associated with this collator's documents.
*/
readonly visibilityPermission?: Permission;
execute(): Promise<IndexableDocument[]>;
}
@@ -88,6 +121,10 @@ export interface DocumentDecorator {
*/
export type QueryTranslator = (query: SearchQuery) => unknown;
export type QueryRequestOptions = {
token?: string;
};
/**
* Interface that must be implemented by specific search engines, responsible
* for performing indexing and querying and translating abstract queries into
@@ -107,5 +144,8 @@ export interface SearchEngine {
/**
* Perform a search query against the SearchEngine.
*/
query(query: SearchQuery): Promise<SearchResultSet>;
query(
query: SearchQuery,
options?: QueryRequestOptions,
): Promise<SearchResultSet>;
}
plugins/catalog-backend/api-report.md
+3
View File
@@ -26,6 +26,7 @@ import { Location as Location_2 } from '@backstage/catalog-model';
import { LocationSpec } from '@backstage/catalog-model';
import { Logger as Logger_2 } from 'winston';
import { Organizations } from 'aws-sdk';
import { Permission } from '@backstage/plugin-permission-common';
import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
import { PermissionCondition } from '@backstage/plugin-permission-common';
import { PermissionCriteria } from '@backstage/plugin-permission-common';
@@ -539,6 +540,8 @@ export class DefaultCatalogCollator implements DocumentCollator {
protected tokenManager: TokenManager;
// (undocumented)
readonly type: string;
// (undocumented)
readonly visibilityPermission: Permission;
}
// Warning: (ae-missing-release-tag) "DefaultCatalogProcessingOrchestrator" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
plugins/catalog-backend/src/search/DefaultCatalogCollator.test.ts
+6
View File
@@ -113,6 +113,9 @@ describe('DefaultCatalogCollator', () => {
componentType: expectedEntities[0]!.spec!.type,
lifecycle: expectedEntities[0]!.spec!.lifecycle,
owner: expectedEntities[0]!.spec!.owner,
authorization: {
resourceRef: 'component:default/test-entity',
},
});
expect(documents[1]).toMatchObject({
title: expectedEntities[1].metadata.title,
@@ -122,6 +125,9 @@ describe('DefaultCatalogCollator', () => {
componentType: expectedEntities[1]!.spec!.type,
lifecycle: expectedEntities[1]!.spec!.lifecycle,
owner: expectedEntities[1]!.spec!.owner,
authorization: {
resourceRef: 'component:default/test-entity-2',
},
});
});
plugins/catalog-backend/src/search/DefaultCatalogCollator.ts
+10 -1
View File
@@ -18,7 +18,11 @@ import {
PluginEndpointDiscovery,
TokenManager,
} from '@backstage/backend-common';
import { Entity, UserEntity } from '@backstage/catalog-model';
import {
Entity,
stringifyEntityRef,
UserEntity,
} from '@backstage/catalog-model';
import { IndexableDocument, DocumentCollator } from '@backstage/search-common';
import { Config } from '@backstage/config';
import {
@@ -26,6 +30,7 @@ import {
CatalogClient,
CatalogEntitiesRequest,
} from '@backstage/catalog-client';
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
export interface CatalogEntityDocument extends IndexableDocument {
componentType: string;
@@ -41,6 +46,7 @@ export class DefaultCatalogCollator implements DocumentCollator {
protected filter?: CatalogEntitiesRequest['filter'];
protected readonly catalogClient: CatalogApi;
public readonly type: string = 'software-catalog';
public readonly visibilityPermission = catalogEntityReadPermission;
protected tokenManager: TokenManager;
static fromConfig(
@@ -126,6 +132,9 @@ export class DefaultCatalogCollator implements DocumentCollator {
kind: entity.kind,
lifecycle: (entity.spec?.lifecycle as string) || '',
owner: (entity.spec?.owner as string) || '',
authorization: {
resourceRef: stringifyEntityRef(entity),
},
};
});
}
plugins/search-backend-node/api-report.md
+3
View File
@@ -5,6 +5,7 @@
```ts
import { DocumentCollator } from '@backstage/search-common';
import { DocumentDecorator } from '@backstage/search-common';
import { DocumentTypeInfo } from '@backstage/search-common';
import { IndexableDocument } from '@backstage/search-common';
import { Logger as Logger_2 } from 'winston';
import { default as lunr_2 } from 'lunr';
@@ -30,6 +31,8 @@ export class IndexBuilder {
scheduler: Scheduler;
}>;
// (undocumented)
getDocumentTypes(): Record<string, DocumentTypeInfo>;
// (undocumented)
getSearchEngine(): SearchEngine;
}
plugins/search-backend-node/src/IndexBuilder.ts
+10
View File
@@ -17,6 +17,7 @@
import {
DocumentCollator,
DocumentDecorator,
DocumentTypeInfo,
IndexableDocument,
SearchEngine,
} from '@backstage/search-common';
@@ -40,12 +41,14 @@ type IndexBuilderOptions = {
export class IndexBuilder {
private collators: Record<string, CollatorEnvelope>;
private decorators: Record<string, DocumentDecorator[]>;
private documentTypes: Record<string, DocumentTypeInfo>;
private searchEngine: SearchEngine;
private logger: Logger;
constructor({ logger, searchEngine }: IndexBuilderOptions) {
this.collators = {};
this.decorators = {};
this.documentTypes = {};
this.logger = logger;
this.searchEngine = searchEngine;
}
@@ -54,6 +57,10 @@ export class IndexBuilder {
return this.searchEngine;
}
getDocumentTypes(): Record<string, DocumentTypeInfo> {
return this.documentTypes;
}
/**
* Makes the index builder aware of a collator that should be executed at the
* given refresh interval.
@@ -69,6 +76,9 @@ export class IndexBuilder {
refreshInterval: defaultRefreshIntervalSeconds,
collate: collator,
};
this.documentTypes[collator.type] = {
visibilityPermission: collator.visibilityPermission,
};
}
/**
plugins/search-backend/api-report.md
+6
View File
@@ -3,8 +3,11 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Config } from '@backstage/config';
import { DocumentTypeInfo } from '@backstage/search-common';
import express from 'express';
import { Logger as Logger_2 } from 'winston';
import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
import { SearchEngine } from '@backstage/plugin-search-backend-node';
// Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -17,6 +20,9 @@ export function createRouter(options: RouterOptions): Promise<express.Router>;
// @public (undocumented)
export type RouterOptions = {
engine: SearchEngine;
types: Record<string, DocumentTypeInfo>;
permissions: PermissionAuthorizer;
config: Config;
logger: Logger_2;
};
```
plugins/search-backend/config.d.ts
Vendored
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface Config {
/** Configuration options for the search plugin */
search?: {
/**
* Options related to the search integration with the Backstage permissions system
*/
permissions?: {
/**
* Limits the amount of time the search backend will spend retrieving and
* authorizing results from the search engine when permissions are
* enabled. When the latency of the query endpoint reaches this threshold,
* the results obtained up until that point will be returned, even if the
* page is incomplete.
*
* This limit is only expected to be hit for broad queries from users with
* extremely restrictive visibility, or for very high page offsets.
*
* @default 1000
*/
queryLatencyBudgetMs?: number;
};
};
}
plugins/search-backend/package.json
+7 -1
View File
@@ -23,12 +23,18 @@
"@backstage/backend-common": "^0.10.4",
"@backstage/config": "^0.1.13",
"@backstage/errors": "^0.2.0",
"@backstage/search-common": "^0.2.0",
"@backstage/search-common": "^0.2.1",
"@backstage/plugin-auth-backend": "^0.7.0-next.0",
"@backstage/plugin-permission-common": "^0.4.0-next.0",
"@backstage/plugin-permission-node": "^0.4.0-next.0",
"@backstage/plugin-search-backend-node": "^0.4.4",
"@backstage/types": "^0.1.1",
"@types/express": "^4.17.6",
"dataloader": "^2.0.0",
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"lodash": "^4.17.21",
"qs": "^6.10.1",
"winston": "^3.2.1",
"yn": "^4.0.0",
"zod": "^3.11.6"
plugins/search-backend/src/service/AuthorizedSearchEngine.test.ts
+577
View File
@@ -0,0 +1,577 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ConfigReader } from '@backstage/config';
import {
AuthorizeDecision,
AuthorizeResult,
PermissionAuthorizer,
} from '@backstage/plugin-permission-common';
import {
DocumentTypeInfo,
IndexableDocument,
SearchEngine,
} from '@backstage/search-common';
import {
encodePageCursor,
decodePageCursor,
AuthorizedSearchEngine,
} from './AuthorizedSearchEngine';
describe('AuthorizedSearchEngine', () => {
const typeUsers = 'users';
const typeTemplates = 'templates';
const typeServices = 'services';
const typeGroups = 'groups';
function generateSampleResults(type: string, withAuthorization?: boolean) {
return Array(10)
.fill(0)
.map((_, index) => ({
type,
document: {
title: `${type}_doc_${index}`,
authorization: withAuthorization
? { resourceRef: `${type}_doc_${index}` }
: undefined,
} as IndexableDocument,
}));
}
const allUsers = generateSampleResults(typeUsers);
const allTemplates = generateSampleResults(typeTemplates);
const results = allUsers.concat(allTemplates);
const mockedQuery: jest.MockedFunction<SearchEngine['query']> = jest.fn();
const searchEngine: SearchEngine = {
setTranslator: () => {
throw new Error('Function not implemented. 1');
},
index: () => {
throw new Error('Function not implemented.2');
},
query: mockedQuery,
};
const mockedAuthorize: jest.MockedFunction<
PermissionAuthorizer['authorize']
> = jest.fn();
const permissionAuthorizer: PermissionAuthorizer = {
authorize: mockedAuthorize,
};
const defaultTypes: Record<string, DocumentTypeInfo> = {
[typeUsers]: {
visibilityPermission: {
name: 'search.users.read',
attributes: { action: 'read' },
},
},
[typeTemplates]: {
visibilityPermission: {
name: 'search.templates.read',
attributes: { action: 'read' },
},
},
[typeServices]: {
visibilityPermission: {
name: 'search.services.read',
attributes: { action: 'read' },
},
},
[typeGroups]: {
visibilityPermission: {
name: 'search.groups.read',
attributes: { action: 'read' },
},
},
};
const authorizedSearchEngine = new AuthorizedSearchEngine(
searchEngine,
defaultTypes,
permissionAuthorizer,
new ConfigReader({}),
);
const options = { token: 'token' };
const allowAll: PermissionAuthorizer['authorize'] = async queries => {
return queries.map(() => ({
result: AuthorizeResult.ALLOW,
}));
};
beforeEach(() => {
mockedQuery.mockReset();
mockedAuthorize.mockClear();
});
it('should forward the parameters correctly', async () => {
mockedQuery.mockImplementation(async () => ({ results }));
mockedAuthorize.mockImplementation(allowAll);
const filters = { just: 1, a: 2, filter: 3 };
await authorizedSearchEngine.query(
{ term: 'term', filters, types: ['one', 'two'] },
options,
);
expect(mockedQuery).toHaveBeenCalledWith(
{
term: 'term',
types: ['one', 'two'],
filters,
},
{ token: 'token' },
);
});
it('should forward the default types if none are passed', async () => {
mockedQuery.mockImplementation(async () => ({ results }));
mockedAuthorize.mockImplementation(allowAll);
await authorizedSearchEngine.query({ term: '' }, options);
expect(mockedQuery).toHaveBeenCalledWith(
{ term: '', types: ['users', 'templates', 'services', 'groups'] },
{ token: 'token' },
);
});
it('should return all the results if all queries are allowed', async () => {
mockedQuery.mockImplementation(async () => ({ results }));
mockedAuthorize.mockImplementation(allowAll);
await expect(
authorizedSearchEngine.query({ term: '' }, options),
).resolves.toEqual({ results });
expect(mockedAuthorize).toHaveBeenCalledTimes(1);
});
it('should batch authorized requests', async () => {
mockedQuery.mockImplementation(async () => ({ results }));
mockedAuthorize.mockImplementation(allowAll);
await authorizedSearchEngine.query(
{ term: '', types: [typeUsers, typeTemplates] },
options,
);
expect(mockedQuery).toHaveBeenCalledWith(
{ term: '', types: ['users', 'templates'] },
{ token: 'token' },
);
expect(mockedAuthorize).toHaveBeenCalledTimes(1);
expect(mockedAuthorize).toHaveBeenLastCalledWith(
[
{ permission: defaultTypes[typeUsers].visibilityPermission },
{ permission: defaultTypes[typeTemplates].visibilityPermission },
],
{ token: 'token' },
);
});
it('should skip sending request for types that are not allowed', async () => {
mockedQuery.mockImplementation(async () => ({ results }));
mockedAuthorize.mockImplementation(async queries => {
return queries.map(query => {
if (
query.permission.name ===
defaultTypes.users.visibilityPermission?.name
) {
return {
result: AuthorizeResult.DENY,
};
}
return {
result: AuthorizeResult.ALLOW,
};
});
});
await authorizedSearchEngine.query({ term: '' }, options);
expect(mockedQuery).toHaveBeenCalledWith(
{ term: '', types: ['templates', 'services', 'groups'] },
{ token: 'token' },
);
expect(mockedAuthorize).toHaveBeenCalledTimes(1);
});
it('should perform result-by-result filtering', async () => {
const usersWithAuth = generateSampleResults(typeUsers, true);
const templatesWithAuth = generateSampleResults(typeTemplates, true);
const resultsWithAuth = usersWithAuth.concat(templatesWithAuth);
mockedQuery.mockImplementation(async () => ({
results: resultsWithAuth,
}));
const userToBeReturned = 8;
mockedAuthorize.mockImplementation(async queries =>
queries.map(query => {
if (
query.permission.name ===
defaultTypes.users.visibilityPermission?.name
) {
if (query.resourceRef) {
return {
result: query.resourceRef.endsWith(userToBeReturned.toString())
? AuthorizeResult.ALLOW
: AuthorizeResult.DENY,
};
}
return {
result: AuthorizeResult.CONDITIONAL,
} as AuthorizeDecision;
}
return {
result: AuthorizeResult.DENY,
};
}),
);
await expect(
authorizedSearchEngine.query({ term: '' }, options),
).resolves.toEqual({ results: [usersWithAuth[userToBeReturned]] });
expect(mockedQuery).toHaveBeenCalledWith(
{ term: '', types: ['users'] },
{ token: 'token' },
);
});
it('should deduplicate authorization queries when resourceRefs match', async () => {
const searchResults = [
{
type: 'templates',
document: {
title: `doc_0_a`,
authorization: { resourceRef: `template_doc_0` },
} as IndexableDocument,
},
{
type: 'templates',
document: {
title: `doc_0_b`,
authorization: { resourceRef: `template_doc_0` },
} as IndexableDocument,
},
];
mockedQuery.mockImplementation(async () => ({
results: searchResults,
}));
mockedAuthorize.mockImplementation(async queries =>
queries.map(query => {
if (query.resourceRef) {
return {
result: AuthorizeResult.ALLOW,
};
}
return {
result: AuthorizeResult.CONDITIONAL,
} as AuthorizeDecision;
}),
);
await expect(
authorizedSearchEngine.query({ term: '', types: ['templates'] }, options),
).resolves.toEqual({ results: searchResults });
expect(mockedAuthorize).toHaveBeenCalledTimes(2);
expect(mockedAuthorize).toHaveBeenNthCalledWith(
1,
[
{
permission: expect.objectContaining({
name: 'search.templates.read',
}),
},
],
{ token: 'token' },
);
expect(mockedAuthorize).toHaveBeenNthCalledWith(
2,
[
{
permission: expect.objectContaining({
name: 'search.templates.read',
}),
resourceRef: 'template_doc_0',
},
],
{ token: 'token' },
);
});
it('should perform search until the target number of results is reached', async () => {
mockedAuthorize.mockImplementation(async queries =>
queries.map(query => {
if (query.resourceRef) {
return {
result: AuthorizeResult.ALLOW,
};
}
return { result: AuthorizeResult.CONDITIONAL } as AuthorizeDecision;
}),
);
const usersWithAuth = generateSampleResults(typeUsers, true);
const templatesWithAuth = generateSampleResults(typeTemplates, true);
const servicesWithAuth = generateSampleResults(typeServices, true);
const allDocuments = [
...usersWithAuth,
...templatesWithAuth,
...servicesWithAuth,
];
mockedQuery
.mockImplementationOnce(async () => ({
results: allDocuments.slice(0, 10),
nextPageCursor: encodePageCursor({ page: 1 }),
}))
.mockImplementationOnce(async () => ({
results: allDocuments.slice(10, 20),
nextPageCursor: encodePageCursor({ page: 2 }),
}))
.mockImplementationOnce(async () => ({
results: allDocuments.slice(20, 30),
}));
const result = await authorizedSearchEngine.query(
{ term: '', types: ['users', 'templates', 'services'] },
options,
);
expect(mockedQuery).toHaveBeenCalledTimes(3);
expect(mockedQuery).toHaveBeenNthCalledWith(
1,
{ term: '', types: ['users', 'templates', 'services'] },
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
2,
{
term: '',
types: ['users', 'templates', 'services'],
pageCursor: 'MQ==',
},
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
3,
{
term: '',
types: ['users', 'templates', 'services'],
pageCursor: 'Mg==',
},
{ token: 'token' },
);
const expectedResult = allDocuments.slice(0, 25);
const expectedFirstRequestCursor = 'MQ==';
expect(result).toEqual({
results: expectedResult,
nextPageCursor: expectedFirstRequestCursor,
});
});
it('should perform search until the target number of results is reached, excluding unauthorized results', async () => {
mockedAuthorize.mockImplementation(async queries =>
queries.map(query => {
if (query.resourceRef) {
return {
result:
query.permission.name === 'search.services.read'
? AuthorizeResult.DENY
: AuthorizeResult.ALLOW,
};
}
return { result: AuthorizeResult.CONDITIONAL } as AuthorizeDecision;
}),
);
const usersWithAuth = generateSampleResults(typeUsers, true);
const templatesWithAuth = generateSampleResults(typeTemplates, true);
const servicesWithAuth = generateSampleResults(typeServices, true);
const groupsWithAuth = generateSampleResults(typeGroups, true);
const allDocuments = [
...usersWithAuth,
...templatesWithAuth,
...servicesWithAuth,
...groupsWithAuth,
].sort(() => Math.floor(Math.random() * 3 - 1));
mockedQuery
.mockImplementationOnce(async () => ({
results: allDocuments.slice(0, 10),
nextPageCursor: encodePageCursor({ page: 1 }),
}))
.mockImplementationOnce(async () => ({
results: allDocuments.slice(10, 20),
nextPageCursor: encodePageCursor({ page: 2 }),
}))
.mockImplementationOnce(async () => ({
results: allDocuments.slice(20, 30),
nextPageCursor: encodePageCursor({ page: 3 }),
}))
.mockImplementationOnce(async () => ({
results: allDocuments.slice(30, 40),
}));
const result = await authorizedSearchEngine.query({ term: '' }, options);
// check if a fourth request is needed for retrieving all results
const fourthRequestNeeded =
allDocuments.slice(0, 30).filter(d => d.type !== typeServices).length <
25;
expect(mockedQuery).toHaveBeenCalledTimes(fourthRequestNeeded ? 4 : 3);
expect(mockedQuery).toHaveBeenNthCalledWith(
1,
{ term: '', types: ['users', 'templates', 'services', 'groups'] },
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
2,
{
term: '',
types: ['users', 'templates', 'services', 'groups'],
pageCursor: 'MQ==',
},
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
3,
{
term: '',
types: ['users', 'templates', 'services', 'groups'],
pageCursor: 'Mg==',
},
{ token: 'token' },
);
const expectedResult = allDocuments
.filter(d => d.type !== typeServices)
.slice(0, 25);
const expectedFirstRequestCursor = 'MQ==';
expect(result).toEqual({
results: expectedResult,
nextPageCursor: expectedFirstRequestCursor,
});
});
it('should discard results until the target cursor is reached', async () => {
mockedAuthorize.mockImplementation(async queries =>
queries.map(query => {
if (query.resourceRef) {
return { result: AuthorizeResult.ALLOW };
}
return { result: AuthorizeResult.CONDITIONAL } as AuthorizeDecision;
}),
);
const usersWithAuth = generateSampleResults(typeUsers, true);
const templatesWithAuth = generateSampleResults(typeTemplates, true);
const servicesWithAuth = generateSampleResults(typeServices, true);
mockedQuery
.mockImplementationOnce(async () => ({
results: usersWithAuth,
nextPageCursor: encodePageCursor({ page: 1 }),
}))
.mockImplementationOnce(async () => ({
results: templatesWithAuth,
nextPageCursor: encodePageCursor({ page: 2 }),
}))
.mockImplementationOnce(async () => ({
results: servicesWithAuth,
}));
const startingFromCursor = encodePageCursor({ page: 1 });
const result = await authorizedSearchEngine.query(
{
term: '',
pageCursor: startingFromCursor,
types: ['users', 'templates', 'services'],
},
options,
);
expect(mockedQuery).toHaveBeenCalledTimes(3);
expect(mockedQuery).toHaveBeenNthCalledWith(
1,
{ term: '', types: ['users', 'templates', 'services'] },
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
2,
{
term: '',
types: ['users', 'templates', 'services'],
pageCursor: 'MQ==',
},
{ token: 'token' },
);
expect(mockedQuery).toHaveBeenNthCalledWith(
3,
{
term: '',
types: ['users', 'templates', 'services'],
pageCursor: 'Mg==',
},
{ token: 'token' },
);
expect(result).toEqual({
results: servicesWithAuth.slice(5),
previousPageCursor: encodePageCursor({ page: 0 }),
});
});
});
describe('decodePageCursor', () => {
it('should correctly decode the cursor', () => {
expect(decodePageCursor()).toEqual({ page: 0 });
expect(decodePageCursor(encodePageCursor({ page: 1 }))).toEqual({
page: 1,
});
expect(decodePageCursor('Mg==')).toEqual({
page: 2,
});
expect(decodePageCursor(encodePageCursor({ page: 0 }))).toEqual({
page: 0,
});
expect(decodePageCursor(encodePageCursor({ page: 100 }))).toEqual({
page: 100,
});
});
it('should throw an error if the cursor is not valid', () => {
expect(() => decodePageCursor(encodePageCursor({ page: -100 }))).toThrow();
expect(() => decodePageCursor('something')).toThrow();
});
});
plugins/search-backend/src/service/AuthorizedSearchEngine.ts
+213
View File
@@ -0,0 +1,213 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { compact, zipObject } from 'lodash';
import qs from 'qs';
import DataLoader from 'dataloader';
import {
AuthorizeDecision,
AuthorizeQuery,
AuthorizeResult,
PermissionAuthorizer,
} from '@backstage/plugin-permission-common';
import {
DocumentTypeInfo,
IndexableDocument,
QueryRequestOptions,
QueryTranslator,
SearchEngine,
SearchQuery,
SearchResult,
SearchResultSet,
} from '@backstage/search-common';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
export function decodePageCursor(pageCursor?: string): { page: number } {
if (!pageCursor) {
return { page: 0 };
}
const page = Number(Buffer.from(pageCursor, 'base64').toString('utf-8'));
if (isNaN(page)) {
throw new InputError('Invalid page cursor');
}
if (page < 0) {
throw new InputError('Invalid page cursor');
}
return {
page,
};
}
export function encodePageCursor({ page }: { page: number }): string {
return Buffer.from(`${page}`, 'utf-8').toString('base64');
}
export class AuthorizedSearchEngine implements SearchEngine {
private readonly pageSize = 25;
private readonly queryLatencyBudgetMs: number;
constructor(
private readonly searchEngine: SearchEngine,
private readonly types: Record<string, DocumentTypeInfo>,
private readonly permissions: PermissionAuthorizer,
config: Config,
) {
this.queryLatencyBudgetMs =
config.getOptionalNumber('search.permissions.queryLatencyBudgetMs') ??
1000;
}
setTranslator(translator: QueryTranslator): void {
this.searchEngine.setTranslator(translator);
}
async index(type: string, documents: IndexableDocument[]): Promise<void> {
this.searchEngine.index(type, documents);
}
async query(
query: SearchQuery,
options: QueryRequestOptions,
): Promise<SearchResultSet> {
const queryStartTime = Date.now();
const authorizer = new DataLoader(
(requests: readonly AuthorizeQuery[]) =>
this.permissions.authorize(requests.slice(), options),
{
// Serialize the permission name and resourceRef as
// a query string to avoid collisions from overlapping
// permission names and resourceRefs.
cacheKeyFn: ({ permission: { name }, resourceRef }) =>
qs.stringify({ name, resourceRef }),
},
);
const requestedTypes = query.types || Object.keys(this.types);
const typeDecisions = zipObject(
requestedTypes,
await Promise.all(
requestedTypes.map(type => {
const permission = this.types[type]?.visibilityPermission;
return permission
? authorizer.load({ permission })
: { result: AuthorizeResult.ALLOW as const };
}),
),
);
const authorizedTypes = requestedTypes.filter(
type => typeDecisions[type]?.result !== AuthorizeResult.DENY,
);
const resultByResultFilteringRequired = authorizedTypes.some(
type => typeDecisions[type]?.result === AuthorizeResult.CONDITIONAL,
);
// When there are no CONDITIONAL decisions for any of the requested
// result types, we can skip filtering result by result by simply
// skipping the types the user is not permitted to see, which will
// be much more efficient.
//
// Since it's not currently possible to configure the page size used
// by search engines, this detail means that a single user might see
// a different page size depending on whether their search required
// result-by-result filtering or not. We can fix this minor
// inconsistency by introducing a configurable page size.
//
// cf. https://github.com/backstage/backstage/issues/9162
if (!resultByResultFilteringRequired) {
return this.searchEngine.query(
{ ...query, types: authorizedTypes },
options,
);
}
const { page } = decodePageCursor(query.pageCursor);
const targetResults = (page + 1) * this.pageSize;
let filteredResults: SearchResult[] = [];
let nextPageCursor: string | undefined;
let latencyBudgetExhausted = false;
do {
const nextPage = await this.searchEngine.query(
{ ...query, types: authorizedTypes, pageCursor: nextPageCursor },
options,
);
filteredResults = filteredResults.concat(
await this.filterResults(nextPage.results, typeDecisions, authorizer),
);
nextPageCursor = nextPage.nextPageCursor;
latencyBudgetExhausted =
Date.now() - queryStartTime > this.queryLatencyBudgetMs;
} while (
nextPageCursor &&
filteredResults.length < targetResults &&
!latencyBudgetExhausted
);
return {
results: filteredResults.slice(
page * this.pageSize,
(page + 1) * this.pageSize,
),
previousPageCursor:
page === 0 ? undefined : encodePageCursor({ page: page - 1 }),
nextPageCursor:
!latencyBudgetExhausted &&
(nextPageCursor || filteredResults.length > targetResults)
? encodePageCursor({ page: page + 1 })
: undefined,
};
}
private async filterResults(
results: SearchResult[],
typeDecisions: Record<string, AuthorizeDecision>,
authorizer: DataLoader<AuthorizeQuery, AuthorizeDecision>,
) {
return compact(
await Promise.all(
results.map(result => {
if (typeDecisions[result.type]?.result === AuthorizeResult.ALLOW) {
return result;
}
const permission = this.types[result.type]?.visibilityPermission;
const resourceRef = result.document.authorization?.resourceRef;
if (!permission || !resourceRef) {
return result;
}
return authorizer
.load({ permission, resourceRef })
.then(decision =>
decision.result === AuthorizeResult.ALLOW ? result : undefined,
);
}),
),
);
}
}
plugins/search-backend/src/service/router.test.ts
+18
View File
@@ -15,6 +15,8 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
import {
IndexBuilder,
LunrSearchEngine,
@@ -25,6 +27,12 @@ import request from 'supertest';
import { createRouter } from './router';
const mockPermissionAuthorizer: PermissionAuthorizer = {
authorize: () => {
throw new Error('Not implemented');
},
};
describe('createRouter', () => {
let app: express.Express;
let mockSearchEngine: jest.Mocked<SearchEngine>;
@@ -36,6 +44,12 @@ describe('createRouter', () => {
const router = await createRouter({
engine: indexBuilder.getSearchEngine(),
types: {
'first-type': {},
'second-type': {},
},
config: new ConfigReader({ permissions: { enabled: false } }),
permissions: mockPermissionAuthorizer,
logger,
});
app = express().use(router);
@@ -74,6 +88,7 @@ describe('createRouter', () => {
'term[0]=foo',
'term[prop]=value',
'types=foo',
'types[0]=unknown-type',
'types[length]=10000&types[0]=first-type',
'filters=stringValue',
'pageCursor[0]=1',
@@ -101,6 +116,9 @@ describe('createRouter', () => {
const router = await createRouter({
engine: indexBuilder.getSearchEngine(),
types: indexBuilder.getDocumentTypes(),
config: new ConfigReader({ permissions: { enabled: false } }),
permissions: mockPermissionAuthorizer,
logger,
});
app = express().use(router);
plugins/search-backend/src/service/router.ts
+19 -4
View File
@@ -20,9 +20,13 @@ import { Logger } from 'winston';
import { z } from 'zod';
import { errorHandler } from '@backstage/backend-common';
import { InputError } from '@backstage/errors';
import { Config } from '@backstage/config';
import { JsonObject, JsonValue } from '@backstage/types';
import { SearchResultSet } from '@backstage/search-common';
import { IdentityClient } from '@backstage/plugin-auth-backend';
import { PermissionAuthorizer } from '@backstage/plugin-permission-common';
import { DocumentTypeInfo, SearchResultSet } from '@backstage/search-common';
import { SearchEngine } from '@backstage/plugin-search-backend-node';
import { AuthorizedSearchEngine } from './AuthorizedSearchEngine';
const jsonObjectSchema: z.ZodSchema<JsonObject> = z.lazy(() => {
const jsonValueSchema: z.ZodSchema<JsonValue> = z.lazy(() =>
@@ -41,6 +45,9 @@ const jsonObjectSchema: z.ZodSchema<JsonObject> = z.lazy(() => {
export type RouterOptions = {
engine: SearchEngine;
types: Record<string, DocumentTypeInfo>;
permissions: PermissionAuthorizer;
config: Config;
logger: Logger;
};
@@ -49,15 +56,21 @@ const allowedLocationProtocols = ['http:', 'https:'];
export async function createRouter(
options: RouterOptions,
): Promise<express.Router> {
const { engine, logger } = options;
const { engine: inputEngine, types, permissions, config, logger } = options;
const requestSchema = z.object({
term: z.string().default(''),
filters: jsonObjectSchema.optional(),
types: z.array(z.string()).optional(),
types: z
.array(z.string().refine(type => Object.keys(types).includes(type)))
.optional(),
pageCursor: z.string().optional(),
});
const engine = config.getOptionalBoolean('permission.enabled')
? new AuthorizedSearchEngine(inputEngine, types, permissions, config)
: inputEngine;
const filterResultSet = ({ results, ...resultSet }: SearchResultSet) => ({
...resultSet,
results: results.filter(result => {
@@ -93,8 +106,10 @@ export async function createRouter(
}, pageCursor=${query.pageCursor ?? ''}`,
);
const token = IdentityClient.getBearerToken(req.header('authorization'));
try {
const resultSet = await engine?.query(query);
const resultSet = await engine?.query(query, { token });
res.send(filterResultSet(resultSet));
} catch (err) {
plugins/search-backend/src/service/standaloneServer.ts
+19 -1
View File
@@ -14,7 +14,12 @@
* limitations under the License.
*/
import { createServiceBuilder } from '@backstage/backend-common';
import {
createServiceBuilder,
loadBackendConfig,
ServerTokenManager,
SingleHostDiscovery,
} from '@backstage/backend-common';
import { Server } from 'http';
import { Logger } from 'winston';
import { createRouter } from './router';
@@ -22,6 +27,7 @@ import {
LunrSearchEngine,
IndexBuilder,
} from '@backstage/plugin-search-backend-node';
import { ServerPermissionClient } from '@backstage/plugin-permission-node';
export interface ServerOptions {
port: number;
@@ -33,14 +39,26 @@ export async function startStandaloneServer(
options: ServerOptions,
): Promise<Server> {
const logger = options.logger.child({ service: 'search-backend' });
const config = await loadBackendConfig({ logger, argv: process.argv });
const searchEngine = new LunrSearchEngine({ logger });
const indexBuilder = new IndexBuilder({ logger, searchEngine });
const discovery = SingleHostDiscovery.fromConfig(config);
const tokenManager = ServerTokenManager.fromConfig(config, {
logger,
});
const permissions = ServerPermissionClient.fromConfig(config, {
discovery,
tokenManager,
});
logger.debug('Starting application server...');
// TODO: stub out some documents/indices?
const router = await createRouter({
engine: indexBuilder.getSearchEngine(),
types: indexBuilder.getDocumentTypes(),
permissions,
config,
logger,
});
plugins/techdocs-backend/api-report.md
+3
View File
@@ -10,6 +10,7 @@ import express from 'express';
import { GeneratorBuilder } from '@backstage/techdocs-common';
import { Knex } from 'knex';
import { Logger as Logger_2 } from 'winston';
import { Permission } from '@backstage/plugin-permission-common';
import { PluginCacheManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PreparerBuilder } from '@backstage/techdocs-common';
@@ -46,6 +47,8 @@ export class DefaultTechDocsCollator implements DocumentCollator {
protected locationTemplate: string;
// (undocumented)
readonly type: string;
// (undocumented)
readonly visibilityPermission: Permission;
}
// Warning: (ae-missing-release-tag) "OutOfTheBoxDeploymentOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
plugins/techdocs-backend/package.json
+1
View File
@@ -37,6 +37,7 @@
"@backstage/config": "^0.1.13",
"@backstage/errors": "^0.2.0",
"@backstage/integration": "^0.7.2",
"@backstage/plugin-catalog-common": "^0.1.1-next.0",
"@backstage/search-common": "^0.2.1",
"@backstage/techdocs-common": "^0.11.4",
"@types/express": "^4.17.6",
plugins/techdocs-backend/src/search/DefaultTechDocsCollator.test.ts
+3
View File
@@ -200,6 +200,9 @@ describe('DefaultTechDocsCollator', () => {
owner: '',
kind: entity.kind,
name: entity.metadata.name,
authorization: {
resourceRef: `component:default/${entity.metadata.name}`,
},
});
});
});
plugins/techdocs-backend/src/search/DefaultTechDocsCollator.ts
+10 -1
View File
@@ -18,13 +18,18 @@ import {
PluginEndpointDiscovery,
TokenManager,
} from '@backstage/backend-common';
import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model';
import {
Entity,
RELATION_OWNED_BY,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { DocumentCollator } from '@backstage/search-common';
import fetch from 'node-fetch';
import unescape from 'lodash/unescape';
import { Logger } from 'winston';
import pLimit from 'p-limit';
import { Config } from '@backstage/config';
import { catalogEntityReadPermission } from '@backstage/plugin-catalog-common';
import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
import { TechDocsDocument } from '@backstage/techdocs-common';
@@ -59,6 +64,7 @@ export class DefaultTechDocsCollator implements DocumentCollator {
private readonly parallelismLimit: number;
private readonly legacyPathCasing: boolean;
public readonly type: string = 'techdocs';
public readonly visibilityPermission = catalogEntityReadPermission;
/**
* @deprecated use static fromConfig method instead.
@@ -148,6 +154,9 @@ export class DefaultTechDocsCollator implements DocumentCollator {
owner:
entity.relations?.find(r => r.type === RELATION_OWNED_BY)
?.target?.name || '',
authorization: {
resourceRef: stringifyEntityRef(entity),
},
}));
} catch (e) {
this.logger.debug(