paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

chore(catalog-backend): make LocationReader, that runs a chain of processors

Browse Source
This commit is contained in:
Fredrik Adelöw
2020-06-04 13:57:38 +02:00
parent ec4aa3d59f
commit fe16d8ee8d
21 changed files with 517 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/src/plugins/catalog.ts
+3 -10
View File
@@ -19,24 +19,17 @@ import {
DatabaseEntitiesCatalog,
DatabaseLocationsCatalog,
DatabaseManager,
DescriptorParsers,
LocationReaders,
IngestionModels,
runPeriodically,
HigherOrderOperations,
LocationReaders,
runPeriodically,
} from '@backstage/plugin-catalog-backend';
import { PluginEnvironment } from '../types';
import { EntityPolicies } from '@backstage/catalog-model';
export default async function createPlugin({
logger,
database,
}: PluginEnvironment) {
const ingestionModel = new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
const ingestionModel = new LocationReaders();
const db = await DatabaseManager.createDatabase(database, logger);
const entitiesCatalog = new DatabaseEntitiesCatalog(db);
packages/backend/tsconfig.json
+1 -1
View File
@@ -9,7 +9,7 @@
"target": "es2019",
"module": "commonjs",
"esModuleInterop": true,
"lib": ["es2019"],
"lib": ["es2019", "dom"],
"types": ["node", "jest"]
}
}
plugins/catalog-backend/src/ingestion/HigherOrderOperations.test.ts
+33 -33
View File
@@ -15,17 +15,17 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { Entity, Location } from '@backstage/catalog-model';
import { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { LocationUpdateStatus } from '../catalog/types';
import { DatabaseLocationUpdateLogStatus } from '../database/types';
import { HigherOrderOperations } from './HigherOrderOperations';
import { IngestionModel } from './types';
import { LocationReader } from './types';
describe('HigherOrderOperations', () => {
let entitiesCatalog: jest.Mocked<EntitiesCatalog>;
let locationsCatalog: jest.Mocked<LocationsCatalog>;
let ingestionModel: jest.Mocked<IngestionModel>;
let locationReader: jest.Mocked<LocationReader>;
let higherOrderOperation: HigherOrderOperations;
beforeAll(() => {
@@ -45,13 +45,13 @@ describe('HigherOrderOperations', () => {
logUpdateSuccess: jest.fn(),
logUpdateFailure: jest.fn(),
};
ingestionModel = {
readLocation: jest.fn(),
locationReader = {
read: jest.fn(),
};
higherOrderOperation = new HigherOrderOperations(
entitiesCatalog,
locationsCatalog,
ingestionModel,
locationReader,
getVoidLogger(),
);
});
@@ -68,7 +68,7 @@ describe('HigherOrderOperations', () => {
};
locationsCatalog.addLocation.mockImplementation(x => Promise.resolve(x));
locationsCatalog.locations.mockResolvedValue([]);
ingestionModel.readLocation.mockResolvedValue([]);
locationReader.read.mockResolvedValue({ entities: [], errors: [] });
const result = await higherOrderOperation.addLocation(spec);
@@ -80,8 +80,8 @@ describe('HigherOrderOperations', () => {
);
expect(result.entities).toEqual([]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(ingestionModel.readLocation).toBeCalledTimes(1);
expect(ingestionModel.readLocation).toBeCalledWith('a', 'b');
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.addOrUpdateEntity).not.toBeCalled();
expect(locationsCatalog.addLocation).toBeCalledTimes(1);
expect(locationsCatalog.addLocation).toBeCalledWith(
@@ -108,15 +108,15 @@ describe('HigherOrderOperations', () => {
data: location,
},
]);
ingestionModel.readLocation.mockResolvedValue([]);
locationReader.read.mockResolvedValue({ entities: [], errors: [] });
const result = await higherOrderOperation.addLocation(spec);
expect(result.location).toEqual(location);
expect(result.entities).toEqual([]);
expect(locationsCatalog.locations).toBeCalledTimes(1);
expect(ingestionModel.readLocation).toBeCalledTimes(1);
expect(ingestionModel.readLocation).toBeCalledWith('a', 'b');
expect(locationReader.read).toBeCalledTimes(1);
expect(locationReader.read).toBeCalledWith({ type: 'a', target: 'b' });
expect(entitiesCatalog.addOrUpdateEntity).not.toBeCalled();
expect(locationsCatalog.addLocation).not.toBeCalled();
});
@@ -126,6 +126,7 @@ describe('HigherOrderOperations', () => {
type: 'a',
target: 'b',
};
const location: LocationSpec = { type: '', target: '' };
const entity: Entity = {
apiVersion: 'a',
kind: 'b',
@@ -133,10 +134,10 @@ describe('HigherOrderOperations', () => {
};
locationsCatalog.locations.mockResolvedValue([]);
ingestionModel.readLocation.mockResolvedValue([
{ type: 'data', data: entity },
{ type: 'error', error: new Error('abcd') },
]);
locationReader.read.mockResolvedValue({
entities: [{ entity, location }],
errors: [{ error: new Error('abcd'), location }],
});
await expect(higherOrderOperation.addLocation(spec)).rejects.toThrow(
/abcd/,
@@ -156,7 +157,7 @@ describe('HigherOrderOperations', () => {
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).not.toHaveBeenCalled();
expect(locationReader.read).not.toHaveBeenCalled();
expect(entitiesCatalog.addOrUpdateEntity).not.toHaveBeenCalled();
});
@@ -181,9 +182,10 @@ describe('HigherOrderOperations', () => {
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockResolvedValue([
{ type: 'data', data: desc },
]);
locationReader.read.mockResolvedValue({
entities: [{ entity: desc, location }],
errors: [],
});
entitiesCatalog.entityByName.mockResolvedValue(undefined);
entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
@@ -192,12 +194,11 @@ describe('HigherOrderOperations', () => {
).resolves.toBeUndefined();
expect(locationsCatalog.locations).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1);
expect(ingestionModel.readLocation).toHaveBeenNthCalledWith(
1,
'some',
'thing',
);
expect(locationReader.read).toHaveBeenCalledTimes(1);
expect(locationReader.read).toHaveBeenNthCalledWith(1, {
type: 'some',
target: 'thing',
});
expect(entitiesCatalog.entityByName).toHaveBeenCalledTimes(1);
expect(entitiesCatalog.entityByName).toHaveBeenNthCalledWith(
1,
@@ -236,9 +237,10 @@ describe('HigherOrderOperations', () => {
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockResolvedValue([
{ type: 'data', data: desc },
]);
locationReader.read.mockResolvedValue({
entities: [{ entity: desc, location }],
errors: [],
});
entitiesCatalog.entityByName.mockResolvedValue(undefined);
entitiesCatalog.addOrUpdateEntity.mockResolvedValue(desc);
@@ -272,15 +274,13 @@ describe('HigherOrderOperations', () => {
locationsCatalog.locations.mockResolvedValue([
{ currentStatus: locationStatus, data: location },
]);
ingestionModel.readLocation.mockRejectedValue(
new Error('reader error message'),
);
locationReader.read.mockRejectedValue(new Error('reader error message'));
await expect(
higherOrderOperation.refreshAllLocations(),
).resolves.toBeUndefined();
expect(ingestionModel.readLocation).toHaveBeenCalledTimes(1);
expect(locationReader.read).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledTimes(1);
expect(locationsCatalog.logUpdateSuccess).not.toHaveBeenCalled();
expect(locationsCatalog.logUpdateFailure).toHaveBeenCalledWith(
plugins/catalog-backend/src/ingestion/HigherOrderOperations.ts
+29 -38
View File
@@ -24,8 +24,11 @@ import {
import lodash from 'lodash';
import { v4 as uuidv4 } from 'uuid';
import { EntitiesCatalog, LocationsCatalog } from '../catalog';
import { IngestionModel } from '../ingestion';
import { AddLocationResult, HigherOrderOperation } from './types';
import {
AddLocationResult,
HigherOrderOperation,
LocationReader,
} from './types';
import { Logger } from 'winston';
/**
@@ -38,18 +41,18 @@ import { Logger } from 'winston';
export class HigherOrderOperations implements HigherOrderOperation {
private readonly entitiesCatalog: EntitiesCatalog;
private readonly locationsCatalog: LocationsCatalog;
private readonly ingestionModel: IngestionModel;
private readonly locationReader: LocationReader;
private readonly logger: Logger;
constructor(
entitiesCatalog: EntitiesCatalog,
locationsCatalog: LocationsCatalog,
ingestionModel: IngestionModel,
locationReader: LocationReader,
logger: Logger,
) {
this.entitiesCatalog = entitiesCatalog;
this.locationsCatalog = locationsCatalog;
this.ingestionModel = ingestionModel;
this.locationReader = locationReader;
this.logger = logger;
}
@@ -80,28 +83,16 @@ export class HigherOrderOperations implements HigherOrderOperation {
};
// Read the location fully, bailing on any errors
const readerOutput = await this.ingestionModel.readLocation(
location.type,
location.target,
);
const inputEntities: Entity[] = [];
for (const entry of readerOutput) {
if (entry.type === 'error') {
throw new InputError(
`Failed to read location ${location.type} ${location.target}, ${entry.error}`,
);
} else {
// Append the location reference annotation
entry.data.metadata.annotations = {
...entry.data.metadata.annotations,
[LOCATION_ANNOTATION]: location.id,
};
inputEntities.push(entry.data);
}
const readerOutput = await this.locationReader.read(spec);
if (readerOutput.errors.length) {
const item = readerOutput.errors[0];
throw new InputError(
`Failed to read location ${item.location.type} ${item.location.target}, ${item.error}`,
);
}
// TODO(freben): At this point, we could detect orphaned entities, by way
// of having a LOCATION_ANNOTATION pointing to the location but not being
// of having a location annotation pointing to the location but not being
// in the entities list. But we aren't sure what to do about those yet.
// Write
@@ -109,9 +100,9 @@ export class HigherOrderOperations implements HigherOrderOperation {
await this.locationsCatalog.addLocation(location);
}
const outputEntities: Entity[] = [];
for (const entity of inputEntities) {
for (const entity of readerOutput.entities) {
const out = await this.entitiesCatalog.addOrUpdateEntity(
entity,
entity.entity,
location.id,
);
outputEntities.push(out);
@@ -157,20 +148,20 @@ export class HigherOrderOperations implements HigherOrderOperation {
// Performs a full refresh of a single location
private async refreshSingleLocation(location: Location) {
const readerOutput = await this.ingestionModel.readLocation(
location.type,
location.target,
);
const readerOutput = await this.locationReader.read({
type: location.type,
target: location.target,
});
for (const readerItem of readerOutput) {
if (readerItem.type === 'error') {
this.logger.debug(
`Failed item in location id="${location.id}" type="${location.type}" target="${location.target}", ${readerItem.error}`,
);
continue;
}
for (const item of readerOutput.errors) {
this.logger.debug(
`Failed item in location type="${item.location.type}" target="${item.location.target}", ${item.error}`,
);
}
for (const item of readerOutput.entities) {
const { entity } = item;
const entity = readerItem.data;
this.logger.debug(
`Read entity kind="${entity.kind}" name="${
entity.metadata.name
plugins/catalog-backend/src/ingestion/IngestionModels.ts
-73
View File
@@ -1,73 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { EntityPolicies, EntityPolicy } from '@backstage/catalog-model';
import { DescriptorParsers } from './descriptor';
import { DescriptorParser, ReaderOutput } from './descriptor/parsers/types';
import { LocationReader, LocationReaders } from './source';
import { IngestionModel } from './types';
export class IngestionModels implements IngestionModel {
private readonly reader: LocationReader;
private readonly parser: DescriptorParser;
private readonly entityPolicy: EntityPolicy;
static default(): IngestionModel {
return new IngestionModels(
new LocationReaders(),
new DescriptorParsers(),
new EntityPolicies(),
);
}
constructor(
reader: LocationReader,
parser: DescriptorParser,
entityPolicy: EntityPolicy,
) {
this.reader = reader;
this.parser = parser;
this.entityPolicy = entityPolicy;
}
async readLocation(type: string, target: string): Promise<ReaderOutput[]> {
const buffer = await this.reader.tryRead(type, target);
if (!buffer) {
throw new Error(`No reader could handle location ${type} ${target}`);
}
const items = await this.parser.tryParse(buffer);
if (!items) {
throw new Error(`No parser could handle location ${type} ${target}`);
}
const result: ReaderOutput[] = [];
for (const item of items) {
if (item.type === 'error') {
result.push(item);
} else {
try {
const output = await this.entityPolicy.enforce(item.data);
result.push({ type: 'data', data: output });
} catch (e) {
result.push({ type: 'error', error: e });
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/LocationReaders.ts
+215
View File
@@ -0,0 +1,215 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotFoundError } from '@backstage/backend-common';
import {
Entity,
EntityPolicies,
EntityPolicy,
LocationSpec,
} from '@backstage/catalog-model';
import { AnnotateLocationEntityProcessor } from './processors/AnnotateLocationEntityProcessor';
import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor';
import { FileReaderProcessor } from './processors/FileReaderProcessor';
import { GithubReaderProcessor } from './processors/GithubReaderProcessor';
import { LocationProcessor, LocationProcessorResult } from './processors/types';
import { YamlProcessor } from './processors/YamlProcessor';
import { LocationReader, ReadLocationResult } from './types';
// The max amount of nesting depth of generated work items
const MAX_DEPTH = 5;
type QueueItem = LocationProcessorResult & { depth: number };
/**
* Implements the reading of a location through a series of processor tasks.
*/
export class LocationReaders implements LocationReader {
private readonly processors: LocationProcessor[];
static defaultProcessors(
entityPolicy: EntityPolicy = new EntityPolicies(),
): LocationProcessor[] {
return [
new FileReaderProcessor(),
new GithubReaderProcessor(),
new YamlProcessor(),
new EntityPolicyProcessor(entityPolicy),
new AnnotateLocationEntityProcessor(),
];
}
constructor(
processors: LocationProcessor[] = LocationReaders.defaultProcessors(),
) {
this.processors = processors;
}
async read(location: LocationSpec): Promise<ReadLocationResult> {
const result: ReadLocationResult = { entities: [], errors: [] };
const queue: QueueItem[] = [];
queue.push({ type: 'location', location, optional: false, depth: 0 });
while (queue.length) {
const entry = queue.shift()!;
const depth = entry.depth + 1;
if (depth > MAX_DEPTH) {
throw new Error(
`Failed to read ${location.type} ${location.target}, max depth exceeded`,
);
}
if (entry.type === 'location') {
await this.handleLocation(entry.location, entry.optional, depth, queue);
} else if (entry.type === 'data') {
await this.handleData(entry.data, entry.location, depth, queue);
} else if (entry.type === 'error') {
await this.handleError(entry.error, entry.location, depth, result);
} else if (entry.type === 'entity') {
await this.handleEntity(
entry.entity,
entry.location,
depth,
queue,
result,
);
}
}
return result;
}
async handleLocation(
location: LocationSpec,
optional: boolean,
depth: number,
queue: QueueItem[],
): Promise<void> {
for (const processor of this.processors) {
try {
const processorOutput = await processor.readLocation?.(location);
if (processorOutput) {
processorOutput.forEach(r => queue.push({ ...r, depth }));
return;
}
} catch (e) {
if (!(e instanceof NotFoundError && optional)) {
queue.push({
type: 'error',
error: e,
location,
depth,
});
}
}
}
queue.push({
type: 'error',
location,
depth,
error: new Error(
`No processor could read location ${location.type} ${location.target}`,
),
});
}
async handleData(
data: Buffer,
location: LocationSpec,
depth: number,
queue: QueueItem[],
): Promise<void> {
for (const processor of this.processors) {
try {
const processorOutput = await processor.parseData?.(data, location);
if (processorOutput) {
processorOutput.forEach(r => queue.push({ ...r, depth }));
return;
}
} catch (e) {
queue.push({ type: 'error', location, error: e, depth });
return;
}
}
queue.push({
type: 'error',
location,
depth,
error: new Error(
`No processor could parse location ${location.type} ${location.target}`,
),
});
}
async handleError(
error: Error,
location: LocationSpec,
_depth: number,
result: ReadLocationResult,
): Promise<void> {
for (const processor of this.processors) {
try {
await processor.handleError?.(error, location);
} catch {
// ignore
}
}
result.errors.push({ location, error });
}
async handleEntity(
entity: Entity,
location: LocationSpec,
depth: number,
queue: QueueItem[],
result: ReadLocationResult,
): Promise<void> {
let resultingEntity = entity;
let foundErrors = false;
for (const processor of this.processors) {
try {
const processorOutput = await processor.processEntity?.(
entity,
location,
);
if (processorOutput) {
resultingEntity = processorOutput;
}
} catch (e) {
foundErrors = true;
queue.push({
type: 'error',
location,
error: e,
depth,
});
}
}
if (!foundErrors) {
result.entities.push({
location,
entity: resultingEntity,
});
}
}
}
plugins/catalog-backend/src/ingestion/descriptor/DescriptorParsers.ts
-45
View File
@@ -1,45 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DescriptorParser, ReaderOutput } from './parsers/types';
import { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
/**
* Parses raw descriptor data (e.g. from a file or stream) into entities.
*/
export class DescriptorParsers implements DescriptorParser {
private readonly parsers: DescriptorParser[];
static defaultParsers(): DescriptorParser[] {
return [new YamlDescriptorParser()];
}
constructor(
parsers: DescriptorParser[] = DescriptorParsers.defaultParsers(),
) {
this.parsers = parsers;
}
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
for (const parser of this.parsers) {
const result = await parser.tryParse(data);
if (result) {
return result;
}
}
throw new Error(`Unsupported descriptor format`);
}
}
plugins/catalog-backend/src/ingestion/descriptor/index.ts
-18
View File
@@ -1,18 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { DescriptorParsers } from './DescriptorParsers';
export { YamlDescriptorParser } from './parsers/YamlDescriptorParser';
plugins/catalog-backend/src/ingestion/descriptor/parsers/YamlDescriptorParser.ts
-64
View File
@@ -1,64 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
import yaml from 'yaml';
import { DescriptorParser, ReaderOutput } from './types';
/**
* Parses descriptors on YAML format
*/
export class YamlDescriptorParser implements DescriptorParser {
async tryParse(data: Buffer): Promise<ReaderOutput[] | undefined> {
// TODO(freben): Should perhaps first do format detection, so the parse
// failure can be emitted as a proper error instead of just as if we
// weren't handling the format at all.
let documents;
try {
documents = yaml.parseAllDocuments(data.toString('utf8'));
} catch (e) {
return undefined;
}
const result: ReaderOutput[] = [];
for (const document of documents) {
if (document.contents) {
if (document.errors?.length) {
result.push({
type: 'error',
error: new Error(`Malformed YAML document, ${document.errors[0]}`),
});
} else {
const json = document.toJSON();
if (typeof json !== 'object' || Array.isArray(json)) {
result.push({
type: 'error',
error: new Error(`Malformed descriptor, expected object at root`),
});
} else {
result.push({
type: 'data',
data: json as Entity,
});
}
}
}
}
return result;
}
}
plugins/catalog-backend/src/ingestion/descriptor/parsers/types.ts
-42
View File
@@ -1,42 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
export type ReaderOutput =
| { type: 'error'; error: Error }
| { type: 'data'; data: Entity };
/**
* Parses raw descriptor data (e.g. from a file) into entities.
*/
export type DescriptorParser = {
/**
* Try to parse some raw data into an entity.
*
* Note that this is only the low level operation of parsing the raw file
* format, e.g. reading JSON or YAML or similar and emitting as structured
* but unvalidated data. The actual validation is performed by EntityPolicy
* and KindParser.
*
* @param data Raw descriptor data
* @returns A list of raw unvalidated entities / errors, or undefined if the
* given data is not meant to be handled by this parser
* @throws An Error if the format was handled and found to not be properly
* formed
*/
tryParse(data: Buffer): Promise<ReaderOutput[] | undefined>;
};
plugins/catalog-backend/src/ingestion/index.ts
+9 -4
View File
@@ -14,8 +14,13 @@
* limitations under the License.
*/
export * from './descriptor';
export { HigherOrderOperations } from './HigherOrderOperations';
export { IngestionModels } from './IngestionModels';
export * from './source';
export type { HigherOrderOperation, IngestionModel } from './types';
export { LocationReaders } from './LocationReaders';
export type {
HigherOrderOperation,
AddLocationResult,
LocationReader,
ReadLocationResult,
ReadLocationEntity,
ReadLocationError,
} from './types';
plugins/catalog-backend/src/ingestion/source/readers/FileLocationReader.ts → plugins/catalog-backend/src/ingestion/processors/AnnotateLocationEntityProcessor.ts
+9 -16
View File
@@ -14,22 +14,15 @@
* limitations under the License.
*/
import fs from 'fs-extra';
import { LocationReader } from './types';
import { Entity, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
import { LocationProcessor } from './types';
/**
* Reads a file from the local file system.
*/
export class FileLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'file') {
return undefined;
}
try {
return await fs.readFile(target);
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
}
export class AnnotateLocationEntityProcessor implements LocationProcessor {
async processEntity(entity: Entity, location: LocationSpec): Promise<Entity> {
const annotations = {
'backstage.io/managed-by-location': `${location.type}:${location.target}`,
};
return lodash.merge({ metadata: { annotations } }, entity);
}
}
plugins/catalog-backend/src/ingestion/source/index.ts → plugins/catalog-backend/src/ingestion/processors/EntityPolicyProcessor.ts
+14 -4
View File
@@ -14,7 +14,17 @@
* limitations under the License.
*/
export { LocationReaders } from './LocationReaders';
export { FileLocationReader } from './readers/FileLocationReader';
export { GitHubLocationReader } from './readers/GitHubLocationReader';
export type { LocationReader } from './readers/types';
import { Entity, EntityPolicy } from '@backstage/catalog-model';
import { LocationProcessor } from './types';
export class EntityPolicyProcessor implements LocationProcessor {
private readonly policy: EntityPolicy;
constructor(policy: EntityPolicy) {
this.policy = policy;
}
async processEntity(entity: Entity): Promise<Entity> {
return this.policy.enforce(entity);
}
}
plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { NotFoundError } from '@backstage/backend-common';
import { LocationSpec } from '@backstage/catalog-model';
import fs from 'fs-extra';
import { LocationProcessor, LocationProcessorResult } from './types';
export class FileReaderProcessor implements LocationProcessor {
async readLocation(
location: LocationSpec,
): Promise<LocationProcessorResult[] | undefined> {
if (location.type !== 'file') {
return undefined;
}
if (!(await fs.pathExists(location.target))) {
throw new NotFoundError(`${location.target} does not exist`);
}
try {
const data = await fs.readFile(location.target);
return [{ type: 'data', location, data }];
} catch (e) {
throw new Error(`Unable to read ${location.target}, ${e}`);
}
}
}
plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.ts → plugins/catalog-backend/src/ingestion/processors/GithubReaderProcessor.ts
+25 -14
View File
@@ -14,30 +14,41 @@
* limitations under the License.
*/
import { NotFoundError } from '@backstage/backend-common';
import { LocationSpec } from '@backstage/catalog-model';
import fetch from 'node-fetch';
import { URL } from 'url';
import { LocationReader } from './types';
import { LocationProcessor, LocationProcessorResult } from './types';
/**
* Reads a file whose target is a GitHub URL.
*
* Uses raw.githubusercontent.com for now, but this will probably change in the
* future when token auth is implemented.
*/
export class GitHubLocationReader implements LocationReader {
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
if (type !== 'github') {
export class GithubReaderProcessor implements LocationProcessor {
async readLocation(
location: LocationSpec,
): Promise<LocationProcessorResult[] | undefined> {
if (location.type !== 'github') {
return undefined;
}
const url = this.buildRawUrl(target);
const url = this.buildRawUrl(location.target);
const response = await fetch(url.toString()); // May also throw
if (!response.ok) {
const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`;
if (response.status === 404) {
throw new NotFoundError(message);
} else {
throw new Error(message);
}
}
try {
return await fetch(url.toString()).then(x => x.buffer());
return [{ type: 'data', location, data: await response.buffer() }];
} catch (e) {
throw new Error(`Unable to read "${target}", ${e}`);
throw new Error(`Unable to read body of ${location.target}, ${e}`);
}
}
// Converts
// from: https://github.com/a/b/blob/master/c.yaml
// to: https://raw.githubusercontent.com/a/b/master/c.yaml
private buildRawUrl(target: string): URL {
try {
const url = new URL(target);
plugins/catalog-backend/src/ingestion/processors/YamlProcessor.ts
+54
View File
@@ -0,0 +1,54 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, LocationSpec } from '@backstage/catalog-model';
import lodash from 'lodash';
import yaml from 'yaml';
import { LocationProcessor, LocationProcessorResult } from './types';
export class YamlProcessor implements LocationProcessor {
async parseData(
data: Buffer,
location: LocationSpec,
): Promise<LocationProcessorResult[] | undefined> {
if (!location.target.match(/\.ya?ml$/)) {
return undefined;
}
let documents: yaml.Document.Parsed[];
try {
documents = yaml.parseAllDocuments(data.toString('utf8')).filter(d => d);
} catch (e) {
const error = new Error(`Failed to parse YAML, ${e}`);
return [{ type: 'error', location, error }];
}
return documents.map(document => {
if (document.errors?.length) {
const error = new Error(`YAML error, ${document.errors[0]}`);
return { type: 'error', location, error };
}
const json = document.toJSON();
if (lodash.isPlainObject(json)) {
return { type: 'entity', location, entity: json as Entity };
}
const error = new Error(`Expected object at root, got ${typeof json}`);
return { type: 'error', location, error };
});
}
}
plugins/catalog-backend/src/ingestion/processors/types.ts
+49
View File
@@ -0,0 +1,49 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { LocationSpec, Entity } from '@backstage/catalog-model';
export type LocationProcessor = {
/**
* Reads the contents of a location.
*
* @param location The location to read
* @returns The output if the location could be read successfully, or
* undefined if the location is not to be handled by this processor
* @throws NotFoundError if the location is handled by this reader, and the
* target did not exist
* @throws Any other Error if the location is handled by this reader, and it
* could not be read successfully
*/
readLocation?(
location: LocationSpec,
): Promise<LocationProcessorResult[] | undefined>;
parseData?(
data: Buffer,
location: LocationSpec,
): Promise<LocationProcessorResult[] | undefined>;
processEntity?(entity: Entity, location: LocationSpec): Promise<Entity>;
handleError?(error: Error, location: LocationSpec): Promise<void>;
};
export type LocationProcessorResult =
| { type: 'error'; error: Error; location: LocationSpec } // An error occurred
| { type: 'location'; location: LocationSpec; optional: boolean } // A location to read
| { type: 'data'; data: Buffer; location: LocationSpec } // Some raw data was read
| { type: 'entity'; entity: Entity; location: LocationSpec }; // An entity was produced
plugins/catalog-backend/src/ingestion/source/LocationReaders.ts
-41
View File
@@ -1,41 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { FileLocationReader } from './readers/FileLocationReader';
import { GitHubLocationReader } from './readers/GitHubLocationReader';
import { LocationReader } from './readers/types';
export class LocationReaders implements LocationReader {
private readonly readers: LocationReader[];
static defaultReaders(): LocationReader[] {
return [new FileLocationReader(), new GitHubLocationReader()];
}
constructor(readers: LocationReader[] = LocationReaders.defaultReaders()) {
this.readers = readers;
}
async tryRead(type: string, target: string): Promise<Buffer | undefined> {
for (const reader of this.readers) {
const result = await reader.tryRead(type, target);
if (result) {
return result;
}
}
throw new Error(`Could not read unknown location "${type}", "${target}"`);
}
}
plugins/catalog-backend/src/ingestion/source/readers/GitHubLocationReader.test.ts
-94
View File
@@ -1,94 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
jest.mock('node-fetch');
import fetch from 'node-fetch';
import { GitHubLocationReader } from './GitHubLocationReader';
const { Response } = jest.requireActual('node-fetch');
describe('Unit: GitHubLocationReader', () => {
beforeEach(() => {
jest.clearAllMocks();
});
it('fetches the file and parses it correctly', async () => {
(fetch as any).mockResolvedValueOnce(new Response('hello'));
const reader = new GitHubLocationReader();
const buffer = await reader.tryRead(
'github',
'https://github.com/spotify/backstage/blob/master/plugins/catalog-backend/fixtures/one_component.yaml',
);
expect(buffer?.toString('utf8')).toBe('hello');
});
it('changes the url to point to https://raw.githubusercontent.com', async () => {
const gitHubUrl = `https://github.com`;
const project = `spotify/backstage`;
const folderPath = `master/plugins/catalog-backend/fixtures`;
const componentFilename = `one_component.yaml`;
const rawGitHubUrl = `https://raw.githubusercontent.com`;
const reader = new GitHubLocationReader();
(fetch as any).mockResolvedValueOnce(new Response('hello'));
await reader.tryRead(
'github',
`${gitHubUrl}/${project}/blob/${folderPath}/${componentFilename}`,
);
expect(fetch).toHaveBeenCalledWith(
`${rawGitHubUrl}/${project}/${folderPath}/${componentFilename}`,
);
});
describe('rejects wrong urls', () => {
const reader = new GitHubLocationReader();
it.each([
['http://example.com/one_component.yaml'],
['http://github.com/one_component.yaml'],
['http://github.com/PROJECT/one_component.yaml'],
['http://github.com/PROJECT/REPO/one_component.yaml'],
['http://github.com/PROJECT/REPO/one_component.json'],
])(
'%p',
async (url: string) =>
await expect(reader.tryRead('github', url)).rejects.toThrow(/url/),
);
});
});
describe('Integration: GitHubLocationSource', () => {
beforeAll(() => {
(fetch as any).mockImplementation(jest.requireActual('node-fetch'));
});
it('fetches the fixture from backstage repo', async () => {
(fetch as any).mockResolvedValueOnce(new Response('component3'));
const PERMANENT_LINK =
'https://github.com/spotify/backstage/blob/ee84a874f8e37f87940cbe515a86c07a2db29541/plugins/catalog-backend/fixtures/one_component.yaml';
const reader = new GitHubLocationReader();
const result = await reader.tryRead('github', PERMANENT_LINK);
expect(result?.toString('utf8')).toContain('component3');
});
});
plugins/catalog-backend/src/ingestion/source/readers/types.ts
-29
View File
@@ -1,29 +0,0 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type LocationReader = {
/**
* Reads the contents of a single location.
*
* @param type The type of location to read
* @param target The location target (type-specific)
* @returns The target contents, as a raw Buffer, or undefined if this type
* was not meant to be consumed by this reader
* @throws An error if the type was meant for this reader, but could not be
* read
*/
tryRead(type: string, target: string): Promise<Buffer | undefined>;
};
plugins/catalog-backend/src/ingestion/types.ts
+35 -6
View File
@@ -15,18 +15,47 @@
*/
import type { Entity, Location, LocationSpec } from '@backstage/catalog-model';
import type { ReaderOutput } from './descriptor/parsers/types';
//
// HigherOrderOperation
//
export type HigherOrderOperation = {
addLocation(spec: LocationSpec): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
};
export type AddLocationResult = {
location: Location;
entities: Entity[];
};
export type IngestionModel = {
readLocation(type: string, target: string): Promise<ReaderOutput[]>;
//
// LocationReader
//
export type LocationReader = {
/**
* Reads the contents of a location.
*
* @param location The location to read
* @throws An error if the location was handled by this reader, but could not
* be read
*/
read(location: LocationSpec): Promise<ReadLocationResult>;
};
export type HigherOrderOperation = {
addLocation(spec: LocationSpec): Promise<AddLocationResult>;
refreshAllLocations(): Promise<void>;
export type ReadLocationResult = {
entities: ReadLocationEntity[];
errors: ReadLocationError[];
};
export type ReadLocationEntity = {
location: LocationSpec;
entity: Entity;
};
export type ReadLocationError = {
location: LocationSpec;
error: Error;
};