paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

Merge pull request #7180 from Erog38/feature/ldap-less-memory

Browse Source 
Reduce LDAP memory usage by releasing ldap SearchEntries from memory during processing.
This commit is contained in:
Fredrik Adelöw
2021-09-17 11:59:20 +02:00
committed by GitHub
parent 13d1b26bd5 65f5317914
commit 4a1612f998
5 changed files with 150 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/proud-adults-bathe.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-ldap': patch
---
Alters LDAP processor to handle one SearchEntry at a time
plugins/catalog-backend-module-ldap/api-report.md
+9
View File
@@ -103,6 +103,15 @@ export class LdapClient {
// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
search(dn: string, options: SearchOptions): Promise<SearchEntry[]>;
// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// Warning: (tsdoc-param-tag-missing-hyphen) The @param block should be followed by a parameter name and then a hyphen
// Warning: (ae-forgotten-export) The symbol "SearchCallback" needs to be exported by the entry point index.d.ts
searchStreaming(
dn: string,
options: SearchOptions,
f: SearchCallback,
): Promise<void>;
}
// Warning: (ae-missing-release-tag) "LdapOrgEntityProvider" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
plugins/catalog-backend-module-ldap/src/ldap/client.ts
+51
View File
@@ -24,6 +24,10 @@ import {
LdapVendor,
} from './vendors';
export interface SearchCallback {
(entry: SearchEntry): void;
}
/**
* Basic wrapper for the ldapjs library.
*
@@ -120,6 +124,53 @@ export class LdapClient {
}
}
/**
* Performs an LDAP search operation, calls a function on each entry to limit memory usage
*
* @param dn The fully qualified base DN to search within
* @param options The search options
* @param f The callback to call on each search entry
*/
async searchStreaming(
dn: string,
options: SearchOptions,
f: SearchCallback,
): Promise<void> {
try {
return await new Promise<void>((resolve, reject) => {
this.client.search(dn, options, (err, res) => {
if (err) {
reject(new Error(errorString(err)));
}
res.on('searchReference', () => {
reject(new Error('Unable to handle referral'));
});
res.on('searchEntry', entry => {
f(entry);
});
res.on('error', e => {
reject(new Error(errorString(e)));
});
res.on('end', r => {
if (!r) {
throw new Error('Null response');
} else if (r.status !== 0) {
throw new Error(`Got status ${r.status}: ${r.errorMessage}`);
} else {
resolve();
}
});
});
});
} catch (e) {
throw new Error(`LDAP search at DN "${dn}" failed, ${e.message}`);
}
}
/**
* Get the Server Vendor.
* Currently only detects Microsoft Active Directory Servers.
plugins/catalog-backend-module-ldap/src/ldap/read.test.ts
+70 -62
View File
@@ -67,7 +67,7 @@ function searchEntry(
describe('readLdapUsers', () => {
const client: jest.Mocked<LdapClient> = {
search: jest.fn(),
searchStreaming: jest.fn(),
getVendor: jest.fn(),
} as any;
@@ -75,18 +75,20 @@ describe('readLdapUsers', () => {
it('transfers all attributes from a default ldap vendor', async () => {
client.getVendor.mockResolvedValue(DefaultLdapVendor);
client.search.mockResolvedValue([
searchEntry({
uid: ['uid-value'],
description: ['description-value'],
cn: ['cn-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
]);
client.searchStreaming.mockImplementation(async (_dn, _opts, fn) => {
await fn(
searchEntry({
uid: ['uid-value'],
description: ['description-value'],
cn: ['cn-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
);
});
const config: UserConfig = {
dn: 'ddd',
options: {},
@@ -129,23 +131,25 @@ describe('readLdapUsers', () => {
it('transfers all attributes from Microsoft Active Directory', async () => {
client.getVendor.mockResolvedValue(ActiveDirectoryVendor);
client.search.mockResolvedValue([
searchEntry({
uid: ['uid-value'],
description: ['description-value'],
cn: ['cn-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
distinguishedName: ['dn-value'],
objectGUID: [
Buffer.from([
68, 2, 125, 190, 209, 0, 94, 73, 133, 33, 230, 174, 234, 195, 160,
152,
]),
],
}),
]);
client.searchStreaming.mockImplementation(async (_dn, _opts, fn) => {
await fn(
searchEntry({
uid: ['uid-value'],
description: ['description-value'],
cn: ['cn-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
distinguishedName: ['dn-value'],
objectGUID: [
Buffer.from([
68, 2, 125, 190, 209, 0, 94, 73, 133, 33, 230, 174, 234, 195, 160,
152,
]),
],
}),
);
});
const config: UserConfig = {
dn: 'ddd',
options: {},
@@ -189,7 +193,7 @@ describe('readLdapUsers', () => {
describe('readLdapGroups', () => {
const client: jest.Mocked<LdapClient> = {
search: jest.fn(),
searchStreaming: jest.fn(),
getVendor: jest.fn(),
} as any;
@@ -197,19 +201,21 @@ describe('readLdapGroups', () => {
it('transfers all attributes from a default ldap vendor', async () => {
client.getVendor.mockResolvedValue(DefaultLdapVendor);
client.search.mockResolvedValue([
searchEntry({
cn: ['cn-value'],
description: ['description-value'],
tt: ['type-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
member: ['e', 'f', 'g'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
]);
client.searchStreaming.mockImplementation(async (_dn, _opts, fn) => {
await fn(
searchEntry({
cn: ['cn-value'],
description: ['description-value'],
tt: ['type-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
member: ['e', 'f', 'g'],
entryDN: ['dn-value'],
entryUUID: ['uuid-value'],
}),
);
});
const config: GroupConfig = {
dn: 'ddd',
options: {},
@@ -260,24 +266,26 @@ describe('readLdapGroups', () => {
});
it('transfers all attributes from Microsoft Active Directory', async () => {
client.getVendor.mockResolvedValue(ActiveDirectoryVendor);
client.search.mockResolvedValue([
searchEntry({
cn: ['cn-value'],
description: ['description-value'],
tt: ['type-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
member: ['e', 'f', 'g'],
distinguishedName: ['dn-value'],
objectGUID: [
Buffer.from([
68, 2, 125, 190, 209, 0, 94, 73, 133, 33, 230, 174, 234, 195, 160,
152,
]),
],
}),
]);
client.searchStreaming.mockImplementation(async (_dn, _opts, fn) => {
await fn(
searchEntry({
cn: ['cn-value'],
description: ['description-value'],
tt: ['type-value'],
mail: ['mail-value'],
avatarUrl: ['avatarUrl-value'],
memberOf: ['x', 'y', 'z'],
member: ['e', 'f', 'g'],
distinguishedName: ['dn-value'],
objectGUID: [
Buffer.from([
68, 2, 125, 190, 209, 0, 94, 73, 133, 33, 230, 174, 234, 195, 160,
152,
]),
],
}),
);
});
const config: GroupConfig = {
dn: 'ddd',
options: {},
plugins/catalog-backend-module-ldap/src/ldap/read.ts
+15 -15
View File
@@ -107,13 +107,11 @@ export async function readLdapUsers(
const transformer = opts?.transformer ?? defaultUserTransformer;
const entries = await client.search(dn, options);
for (const user of entries) {
await client.searchStreaming(dn, options, async user => {
const entity = await transformer(vendor, config, user);
if (!entity) {
continue;
return;
}
mapReferencesAttr(user, vendor, map.memberOf, (myDn, vs) => {
@@ -121,7 +119,7 @@ export async function readLdapUsers(
});
entities.push(entity);
}
});
return { users: entities, userMemberOf };
}
@@ -210,24 +208,26 @@ export async function readLdapGroups(
const transformer = opts?.transformer ?? defaultGroupTransformer;
const entries = await client.search(dn, options);
for (const group of entries) {
const entity = await transformer(vendor, config, group);
if (!entity) {
continue;
await client.searchStreaming(dn, options, async entry => {
if (!entry) {
return;
}
mapReferencesAttr(group, vendor, map.memberOf, (myDn, vs) => {
const entity = await transformer(vendor, config, entry);
if (!entity) {
return;
}
mapReferencesAttr(entry, vendor, map.memberOf, (myDn, vs) => {
ensureItems(groupMemberOf, myDn, vs);
});
mapReferencesAttr(group, vendor, map.members, (myDn, vs) => {
mapReferencesAttr(entry, vendor, map.members, (myDn, vs) => {
ensureItems(groupMember, myDn, vs);
});
groups.push(entity);
}
});
return {
groups,