paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

catalog-backend: remove data parsing processing step

Browse Source
This commit is contained in:
Patrik Oldsberg
2020-10-26 17:05:07 +01:00
parent 36a71d2780
commit 2f62e18041
14 changed files with 250 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/mighty-starfishes-taste.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': minor
---
Removed the parseData step from catalog processors. Locations readers should emit full entities instead.
plugins/catalog-backend/src/ingestion/LocationReaders.ts
-37
View File
@@ -28,7 +28,6 @@ import { CatalogRulesEnforcer } from './CatalogRules';
import * as result from './processors/results';
import {
CatalogProcessor,
CatalogProcessorDataResult,
CatalogProcessorEmit,
CatalogProcessorEntityResult,
CatalogProcessorErrorResult,
@@ -75,8 +74,6 @@ export class LocationReaders implements LocationReader {
for (const item of items) {
if (item.type === 'location') {
await this.handleLocation(item, emit);
} else if (item.type === 'data') {
await this.handleData(item, emit);
} else if (item.type === 'entity') {
if (rulesEnforcer.isAllowed(item.entity, item.location)) {
const relations = Array<EntityRelationSpec>();
@@ -165,40 +162,6 @@ export class LocationReaders implements LocationReader {
logger.warn(message);
}
private async handleData(
item: CatalogProcessorDataResult,
emit: CatalogProcessorEmit,
) {
const { processors, logger } = this.options;
const validatedEmit: CatalogProcessorEmit = emitResult => {
if (emitResult.type === 'relation') {
throw new Error('parseData may not emit entity relations');
}
emit(emitResult);
};
for (const processor of processors) {
if (processor.parseData) {
try {
if (
await processor.parseData(item.data, item.location, validatedEmit)
) {
return;
}
} catch (e) {
const message = `Processor ${processor.constructor.name} threw an error while parsing ${item.location.type} ${item.location.target}, ${e}`;
emit(result.generalError(item.location, message));
logger.warn(message);
}
}
}
const message = `No processor was able to parse location ${item.location.type} ${item.location.target}`;
emit(result.inputError(item.location, message));
}
private async handleEntity(
item: CatalogProcessorEntityResult,
emit: CatalogProcessorEmit,
plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts
+5 -1
View File
@@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model';
import fs from 'fs-extra';
import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { parseEntityYaml } from './util/parse';
export class FileReaderProcessor implements CatalogProcessor {
async readLocation(
@@ -33,7 +34,10 @@ export class FileReaderProcessor implements CatalogProcessor {
const exists = await fs.pathExists(location.target);
if (exists) {
const data = await fs.readFile(location.target);
emit(result.data(location, data));
for (const parseResult of parseEntityYaml(data, location)) {
emit(parseResult);
}
} else if (!optional) {
const message = `${location.type} ${location.target} does not exist`;
emit(result.notFoundError(location, message));
plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.test.ts
+5 -5
View File
@@ -21,7 +21,7 @@ import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { msw } from '@backstage/test-utils';
import {
CatalogProcessorDataResult,
CatalogProcessorEntityResult,
CatalogProcessorErrorResult,
CatalogProcessorResult,
} from './types';
@@ -42,17 +42,17 @@ describe('UrlReaderProcessor', () => {
server.use(
rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) =>
res(ctx.body('Hello')),
res(ctx.json({ mock: 'entity' })),
),
);
const generated = (await new Promise<CatalogProcessorResult>(emit =>
processor.readLocation(spec, false, emit),
)) as CatalogProcessorDataResult;
)) as CatalogProcessorEntityResult;
expect(generated.type).toBe('data');
expect(generated.type).toBe('entity');
expect(generated.location).toBe(spec);
expect(generated.data.toString('utf8')).toBe('Hello');
expect(generated.entity).toEqual({ mock: 'entity' });
});
it('should fail load from url with error', async () => {
plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts
+5 -1
View File
@@ -19,6 +19,7 @@ import { LocationSpec } from '@backstage/catalog-model';
import { Logger } from 'winston';
import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { parseEntityYaml } from './util/parse';
// TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this
// can be removed in a bit
@@ -55,7 +56,10 @@ export class UrlReaderProcessor implements CatalogProcessor {
try {
const data = await this.options.reader.read(location.target);
emit(result.data(location, data));
for (const parseResult of parseEntityYaml(data, location)) {
emit(parseResult);
}
} catch (error) {
const message = `Unable to read ${location.type}, ${error}`;
plugins/catalog-backend/src/ingestion/processors/YamlProcessor.test.ts
-151
View File
@@ -1,151 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import { TextEncoder } from 'util';
import yaml from 'yaml';
import {
CatalogProcessorEntityResult,
CatalogProcessorErrorResult,
} from './types';
import { YamlProcessor } from './YamlProcessor';
describe('YamlProcessor', () => {
const processor = new YamlProcessor();
const locationSpec = {
type: 'url',
target: 'http://example.com/component.yaml',
};
function encodeEntity(entity: string): Buffer {
const data = new TextEncoder().encode(entity);
return Buffer.from(data);
}
it('should only process files with yaml', async () => {
const wrongLocationSpec = {
type: 'url',
target: 'http://example.com/component.json',
};
const buffer = Buffer.from([]);
const never = jest.fn();
expect(await processor.parseData(buffer, wrongLocationSpec, never)).toBe(
false,
);
expect(never).not.toBeCalled();
});
it('should process url that contains yaml', async () => {
const containsYamlLocationSpec = {
type: 'url',
target: 'http://example.com/component?path=test.yaml&c=1&d=2',
};
const buffer = Buffer.from([]);
const emit = jest.fn();
expect(
await processor.parseData(buffer, containsYamlLocationSpec, emit),
).toBe(true);
expect(emit).toBeCalled();
});
it('should process entity with yaml', async () => {
const entity = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'my-component',
},
spec: {},
} as Entity;
const buffer = encodeEntity(yaml.stringify(entity));
const emit = jest.fn();
expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true);
const e = emit.mock.calls[0][0] as CatalogProcessorEntityResult;
expect(e.type).toBe('entity');
expect(e.location).toBe(locationSpec);
expect(e.entity).toEqual(entity);
});
it('should process multiple entities with yaml', async () => {
const entityComponent = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'my-component',
},
spec: {},
} as Entity;
const entityApi = {
apiVersion: 'backstage.io/v1alpha1',
kind: 'API',
metadata: {
name: 'my-api',
},
spec: {},
} as Entity;
const buffer = encodeEntity(
`${yaml.stringify(entityComponent)}---\n${yaml.stringify(entityApi)}`,
);
const emit = jest.fn();
expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true);
const eComponent = emit.mock.calls[0][0] as CatalogProcessorEntityResult;
expect(eComponent.type).toBe('entity');
expect(eComponent.location).toBe(locationSpec);
expect(eComponent.entity).toEqual(entityComponent);
const eApi = emit.mock.calls[1][0] as CatalogProcessorEntityResult;
expect(eApi.type).toBe('entity');
expect(eApi.location).toBe(locationSpec);
expect(eApi.entity).toEqual(entityApi);
});
it('should fail process entity on invalid yaml', async () => {
const buffer = encodeEntity('{');
const emit = jest.fn();
expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true);
const e = emit.mock.calls[0][0] as CatalogProcessorErrorResult;
expect(e.error.message).toMatch(/^YAML error, /);
expect(e.type).toBe('error');
expect(e.location).toBe(locationSpec);
});
it('should fail process entity if not object at root', async () => {
const buffer = encodeEntity('[]');
const emit = jest.fn();
expect(await processor.parseData(buffer, locationSpec, emit)).toBe(true);
const e = emit.mock.calls[0][0] as CatalogProcessorErrorResult;
expect(e.error.message).toMatch(/^Expected object at root, got /);
expect(e.type).toBe('error');
expect(e.location).toBe(locationSpec);
});
});
plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts
-63
View File
@@ -1,63 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
import yaml from 'yaml';
import * as result from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
/**
* Handles incoming raw data buffers, and if they have a yaml extension,
* attempts to parse them into structured data and emitting them as un-
* validated entities.
*/
export class YamlProcessor implements CatalogProcessor {
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<boolean> {
if (!location.target.match(/\.ya?ml/)) {
return false;
}
let documents: yaml.Document.Parsed[];
try {
documents = yaml.parseAllDocuments(data.toString('utf8')).filter(d => d);
} catch (e) {
emit(result.generalError(location, `Failed to parse YAML, ${e}`));
return true;
}
for (const document of documents) {
if (document.errors?.length) {
const message = `YAML error, ${document.errors[0]}`;
emit(result.generalError(location, message));
} else {
const json = document.toJSON();
if (lodash.isPlainObject(json)) {
emit(result.entity(location, json as Entity));
} else {
const message = `Expected object at root, got ${typeof json}`;
emit(result.generalError(location, message));
}
}
}
return true;
}
}
plugins/catalog-backend/src/ingestion/processors/index.ts
-1
View File
@@ -29,4 +29,3 @@ export { PlaceholderProcessor } from './PlaceholderProcessor';
export type { PlaceholderResolver } from './PlaceholderProcessor';
export { StaticLocationProcessor } from './StaticLocationProcessor';
export { UrlReaderProcessor } from './UrlReaderProcessor';
export { YamlProcessor } from './YamlProcessor';
plugins/catalog-backend/src/ingestion/processors/results.ts
-7
View File
@@ -51,13 +51,6 @@ export function generalError(
return { type: 'error', location: atLocation, error: new Error(message) };
}
export function data(
atLocation: LocationSpec,
newData: Buffer,
): CatalogProcessorResult {
return { type: 'data', location: atLocation, data: newData };
}
export function location(
newLocation: LocationSpec,
optional: boolean,
plugins/catalog-backend/src/ingestion/processors/types.ts
-21
View File
@@ -35,20 +35,6 @@ export type CatalogProcessor = {
emit: CatalogProcessorEmit,
): Promise<boolean>;
/**
* Parses a raw data buffer that was read from a location.
*
* @param data The data to parse
* @param location The location that the data came from
* @param emit A sink for items resulting from the parsing
* @returns True if handled by this processor, false otherwise
*/
parseData?(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<boolean>;
/**
* Pre-processes an emitted entity, after it has been emitted but before it
* has been validated.
@@ -105,12 +91,6 @@ export type CatalogProcessorLocationResult = {
optional: boolean;
};
export type CatalogProcessorDataResult = {
type: 'data';
data: Buffer;
location: LocationSpec;
};
export type CatalogProcessorEntityResult = {
type: 'entity';
entity: Entity;
@@ -131,7 +111,6 @@ export type CatalogProcessorErrorResult = {
export type CatalogProcessorResult =
| CatalogProcessorLocationResult
| CatalogProcessorDataResult
| CatalogProcessorEntityResult
| CatalogProcessorRelationResult
| CatalogProcessorErrorResult;
plugins/catalog-backend/src/ingestion/processors/util/parse.test.ts
+180
View File
@@ -0,0 +1,180 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseEntityYaml } from './parse';
import * as result from '../results';
const testLoc = {
target: 'my-loc-target',
type: 'my-loc-type',
};
describe('parseEntityYaml', () => {
it('should parse a yaml', () => {
const results = Array.from(
parseEntityYaml(
Buffer.from(
`
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: backstage
description: backstage.io
annotations:
github.com/project-slug: 'spotify/backstage'
spec:
type: website
lifecycle: production
owner: guest
`,
'utf8',
),
testLoc,
),
);
expect(results).toEqual([
result.entity(testLoc, {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'backstage',
description: 'backstage.io',
annotations: {
'github.com/project-slug': 'spotify/backstage',
},
},
spec: {
type: 'website',
lifecycle: 'production',
owner: 'guest',
},
}),
]);
});
it('should parse multiple docs', () => {
const results = Array.from(
parseEntityYaml(
Buffer.from(
`
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: web
spec:
type: website
---
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: srv
spec:
type: service
`,
'utf8',
),
testLoc,
),
);
expect(results).toEqual([
result.entity(testLoc, {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'web',
},
spec: {
type: 'website',
},
}),
result.entity(testLoc, {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'srv',
},
spec: {
type: 'service',
},
}),
]);
});
it('should emit parsing errors', () => {
const results = Array.from(
parseEntityYaml(Buffer.from('`', 'utf8'), testLoc),
);
// Parse errors are always per document
expect(results).toEqual([
result.generalError(
testLoc,
'YAML error, YAMLSemanticError: Plain value cannot start with reserved character `',
),
]);
});
it('should emit parsing errors for individual documents', () => {
const results = Array.from(
parseEntityYaml(
Buffer.from(
`
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: web
spec:
type: website
---
apiVersion: backstage.io/v1alpha1
this: - is - not [valid] yaml
`,
'utf8',
),
testLoc,
),
);
expect(results).toEqual([
result.entity(testLoc, {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Component',
metadata: {
name: 'web',
},
spec: {
type: 'website',
},
}),
result.generalError(
testLoc,
'YAML error, YAMLSemanticError: Nested mappings are not allowed in compact mappings',
),
]);
});
it('must be an object at root', () => {
const results = Array.from(
parseEntityYaml(Buffer.from('imma-string', 'utf8'), testLoc),
);
expect(results).toEqual([
result.generalError(testLoc, 'Expected object at root, got string'),
]);
});
});
plugins/catalog-backend/src/ingestion/processors/util/parse.ts
+49
View File
@@ -0,0 +1,49 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
import yaml from 'yaml';
import * as result from '../results';
import { CatalogProcessorResult } from '../types';
export function* parseEntityYaml(
data: Buffer,
location: LocationSpec,
): Iterable<CatalogProcessorResult> {
let documents: yaml.Document.Parsed[];
try {
documents = yaml.parseAllDocuments(data.toString('utf8')).filter(d => d);
} catch (e) {
yield result.generalError(location, `Failed to parse YAML, ${e}`);
return;
}
for (const document of documents) {
if (document.errors?.length) {
const message = `YAML error, ${document.errors[0]}`;
yield result.generalError(location, message);
} else {
const json = document.toJSON();
if (lodash.isPlainObject(json)) {
yield result.entity(location, json as Entity);
} else {
const message = `Expected object at root, got ${typeof json}`;
yield result.generalError(location, message);
}
}
}
}
plugins/catalog-backend/src/service/CatalogBuilder.test.ts
+1 -10
View File
@@ -83,15 +83,6 @@ describe('CatalogBuilder', () => {
emit: CatalogProcessorEmit,
) {
expect(location.type).toBe('test');
emit(result.data(location, await reader.read('ignored')));
return true;
},
async parseData(
data: Buffer,
location: LocationSpec,
emit: CatalogProcessorEmit,
) {
expect(data.toString()).toEqual('junk');
emit(
result.entity(location, {
apiVersion: 'av',
@@ -123,7 +114,7 @@ describe('CatalogBuilder', () => {
type: 'test',
target: '',
});
expect.assertions(8);
expect.assertions(7);
expect(added.entities).toEqual([
{
apiVersion: 'av',
plugins/catalog-backend/src/service/CatalogBuilder.ts
-2
View File
@@ -56,7 +56,6 @@ import {
PlaceholderResolver,
StaticLocationProcessor,
UrlReaderProcessor,
YamlProcessor,
} from '../ingestion';
import { CatalogRulesEnforcer } from '../ingestion/CatalogRules';
import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor';
@@ -329,7 +328,6 @@ export class CatalogBuilder {
GithubOrgReaderProcessor.fromConfig(config, { logger }),
LdapOrgReaderProcessor.fromConfig(config, { logger }),
new UrlReaderProcessor({ reader, logger }),
new YamlProcessor(),
new CodeOwnersProcessor({ reader }),
new LocationRefProcessor(),
new OwnerRelationProcessor(),