paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

Merge pull request #12156 from RoadieHQ/store-refresh-keys-to-entities

Browse Source 
feat: add support for refreshKeys
This commit is contained in:
Patrik Oldsberg
2022-07-07 14:11:11 +02:00
committed by GitHub
parent aa2877058b 1c23763ce8
commit 9a03e9c71d
26 changed files with 370 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/cold-coins-tickle.md
+19
View File
@@ -0,0 +1,19 @@
---
'@backstage/plugin-catalog-backend': minor
---
Added an option to be able to trigger refreshes on entities based on a prestored arbitrary key.
The UrlReaderProcessor, FileReaderProcessor got updated to store the absolute URL of the catalog file as a refresh key. In the format of `<type>:<target>`
The PlaceholderProcessor got updated to store the resolverValues as refreshKeys for the entities.
The custom resolvers will need to be updated to pass in a `CatalogProcessorEmit` function as parameter and they should be updated to emit their refresh processingResults. You can see the updated resolvers in the `PlaceholderProcessor.ts`
```ts
// yamlPlaceholderResolver
...
const { content, url } = await readTextLocation(params);
params.emit(processingResult.refresh(`url:${url}`));
...
```
plugins/catalog-backend/api-report.md
+15 -2
View File
@@ -268,6 +268,12 @@ export type CatalogProcessorParser = (options: {
location: LocationSpec;
}) => AsyncIterable<CatalogProcessorResult>;
// @public (undocumented)
export type CatalogProcessorRefreshKeysResult = {
type: 'refresh';
key: string;
};
// @public (undocumented)
export type CatalogProcessorRelationResult = {
type: 'relation';
@@ -279,7 +285,8 @@ export type CatalogProcessorResult =
| CatalogProcessorLocationResult
| CatalogProcessorEntityResult
| CatalogProcessorRelationResult
| CatalogProcessorErrorResult;
| CatalogProcessorErrorResult
| CatalogProcessorRefreshKeysResult;
// @public (undocumented)
export class CodeOwnersProcessor implements CatalogProcessor {
@@ -545,7 +552,11 @@ export class PlaceholderProcessor implements CatalogProcessor {
// (undocumented)
getProcessorName(): string;
// (undocumented)
preProcessEntity(entity: Entity, location: LocationSpec): Promise<Entity>;
preProcessEntity(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity>;
}
// @public (undocumented)
@@ -567,6 +578,7 @@ export type PlaceholderResolverParams = {
baseUrl: string;
read: PlaceholderResolverRead;
resolveUrl: PlaceholderResolverResolveUrl;
emit: CatalogProcessorEmit;
};
// @public (undocumented)
@@ -601,6 +613,7 @@ export const processingResult: Readonly<{
newEntity: Entity,
) => CatalogProcessorResult;
readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult;
readonly refresh: (key: string) => CatalogProcessorResult;
}>;
// @public (undocumented)
plugins/catalog-backend/migrations/20220616202842_refresh_keys.js
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @param { import("knex").Knex } knex
*/
exports.up = async function up(knex) {
await knex.schema.createTable('refresh_keys', table => {
table.comment(
'This table contains relations between entities and keys to trigger refreshes with',
);
table
.text('entity_id')
.notNullable()
.references('entity_id')
.inTable('refresh_state')
.onDelete('CASCADE')
.comment('A reference to the entity that the refresh key is tied to');
table
.text('key')
.notNullable()
.comment(
'A reference to a key which should be used to trigger a refresh on this entity',
);
table.index('entity_id', 'refresh_keys_entity_id_idx');
table.index('key', 'refresh_keys_key_idx');
});
};
/**
* @param { import("knex").Knex } knex
*/
exports.down = async function down(knex) {
await knex.schema.alterTable('refresh_keys', table => {
table.dropIndex([], 'refresh_keys_entity_id_idx');
table.dropIndex([], 'refresh_keys_key_idx');
});
await knex.schema.dropTable('refresh_keys');
};
plugins/catalog-backend/src/api/index.ts
+1
View File
@@ -26,6 +26,7 @@ export type {
CatalogProcessorRelationResult,
CatalogProcessorErrorResult,
CatalogProcessorResult,
CatalogProcessorRefreshKeysResult,
} from './processor';
export type {
EntityProvider,
plugins/catalog-backend/src/api/processingResult.ts
+4
View File
@@ -65,4 +65,8 @@ export const processingResult = Object.freeze({
relation(spec: EntityRelationSpec): CatalogProcessorResult {
return { type: 'relation', relation: spec };
},
refresh(key: string): CatalogProcessorResult {
return { type: 'refresh', key };
},
} as const);
plugins/catalog-backend/src/api/processor.ts
+8 -1
View File
@@ -169,9 +169,16 @@ export type CatalogProcessorErrorResult = {
location: LocationSpec;
};
/** @public */
export type CatalogProcessorRefreshKeysResult = {
type: 'refresh';
key: string;
};
/** @public */
export type CatalogProcessorResult =
| CatalogProcessorLocationResult
| CatalogProcessorEntityResult
| CatalogProcessorRelationResult
| CatalogProcessorErrorResult;
| CatalogProcessorErrorResult
| CatalogProcessorRefreshKeysResult;
plugins/catalog-backend/src/database/DefaultProcessingDatabase.test.ts
+66
View File
@@ -24,6 +24,7 @@ import { DateTime } from 'luxon';
import { applyDatabaseMigrations } from './migrations';
import { DefaultProcessingDatabase } from './DefaultProcessingDatabase';
import {
DbRefreshKeysRow,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbRelationsRow,
@@ -98,6 +99,7 @@ describe('Default Processing Database', () => {
resultHash: '',
relations: [],
deferredEntities: [],
refreshKeys: [],
}),
).rejects.toThrow(
`Conflicting write of processing result for ${id} with location key 'undefined'`,
@@ -117,6 +119,7 @@ describe('Default Processing Database', () => {
relations: [],
deferredEntities: [],
locationKey: 'key',
refreshKeys: [],
errors: "['something broke']",
};
const { knex, db } = await createDatabase(databaseId);
@@ -143,6 +146,7 @@ describe('Default Processing Database', () => {
...options,
resultHash: '',
locationKey: 'fail',
refreshKeys: [],
}),
).rejects.toThrow(
`Conflicting write of processing result for ${id} with location key 'fail'`,
@@ -174,6 +178,7 @@ describe('Default Processing Database', () => {
relations: [],
deferredEntities: [],
locationKey: 'key',
refreshKeys: [],
errors: "['something broke']",
}),
);
@@ -228,6 +233,7 @@ describe('Default Processing Database', () => {
resultHash: '',
relations: relations,
deferredEntities: [],
refreshKeys: [],
}),
);
@@ -250,6 +256,7 @@ describe('Default Processing Database', () => {
resultHash: '',
relations: relations,
deferredEntities: [],
refreshKeys: [],
}),
);
@@ -309,6 +316,7 @@ describe('Default Processing Database', () => {
resultHash: '',
relations: [],
deferredEntities,
refreshKeys: [],
}),
);
@@ -400,6 +408,7 @@ describe('Default Processing Database', () => {
processedEntity,
resultHash: '',
relations: [],
refreshKeys: [],
deferredEntities: [
{
entity: {
@@ -465,6 +474,63 @@ describe('Default Processing Database', () => {
},
60_000,
);
it.each(databases.eachSupportedId())(
'stores the refresh keys for the entity',
async databaseId => {
const mockLogger = {
debug: jest.fn(),
error: jest.fn(),
warn: jest.fn(),
};
const { knex, db } = await createDatabase(
databaseId,
mockLogger as unknown as Logger,
);
await insertRefreshStateRow(knex, {
entity_id: id,
entity_ref: 'location:default/fakelocation',
unprocessed_entity: '{}',
processed_entity: '{}',
errors: '[]',
next_update_at: '2021-04-01 13:37:00',
last_discovery_at: '2021-04-01 13:37:00',
});
const deferredEntities = [
{
entity: {
apiVersion: '1',
kind: 'Location',
metadata: {
name: 'next',
},
},
locationKey: 'mock',
},
];
await db.transaction(tx =>
db.updateProcessedEntity(tx, {
id,
processedEntity,
resultHash: '',
relations: [],
deferredEntities,
refreshKeys: [{ key: 'protocol:foo-bar.com' }],
}),
);
const refreshKeys = await knex<DbRefreshKeysRow>('refresh_keys')
.where({ entity_id: id })
.select();
expect(refreshKeys[0]).toEqual({
entity_id: id,
key: 'protocol:foo-bar.com',
});
},
);
});
describe('updateEntityCache', () => {
plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts
+39 -1
View File
@@ -33,12 +33,14 @@ import {
UpdateEntityCacheOptions,
ListParentsOptions,
ListParentsResult,
RefreshByKeyOptions,
} from './types';
import { DeferredEntity } from '../processing/types';
import { ProcessingIntervalFunction } from '../processing/refresh';
import { rethrowError, timestampToDateTime } from './conversion';
import { initDatabaseMetrics } from './metrics';
import {
DbRefreshKeysRow,
DbRefreshStateReferencesRow,
DbRefreshStateRow,
DbRelationsRow,
@@ -77,6 +79,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
errors,
relations,
deferredEntities,
refreshKeys,
locationKey,
} = options;
const refreshResult = await tx<DbRefreshStateRow>('refresh_state')
@@ -100,11 +103,12 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
`Conflicting write of processing result for ${id} with location key '${locationKey}'`,
);
}
const sourceEntityRef = stringifyEntityRef(processedEntity);
// Schedule all deferred entities for future processing.
await this.addUnprocessedEntities(tx, {
entities: deferredEntities,
sourceEntityRef: stringifyEntityRef(processedEntity),
sourceEntityRef,
});
// Delete old relations
@@ -138,6 +142,21 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
BATCH_SIZE,
);
// Delete old refresh keys
await tx<DbRefreshKeysRow>('refresh_keys')
.where({ entity_id: id })
.delete();
// Insert the refresh keys for the processed entity
await tx.batchInsert(
'refresh_keys',
refreshKeys.map(k => ({
entity_id: id,
key: k.key,
})),
BATCH_SIZE,
);
return {
previous: {
relations: previousRelationRows,
@@ -516,6 +535,25 @@ export class DefaultProcessingDatabase implements ProcessingDatabase {
}
}
async refreshByRefreshKeys(
txOpaque: Transaction,
options: RefreshByKeyOptions,
) {
const tx = txOpaque as Knex.Transaction;
const { keys } = options;
await tx<DbRefreshStateRow>('refresh_state')
.whereIn('entity_id', function selectEntityRefs(tx2) {
tx2
.whereIn('key', keys)
.select({
entity_id: 'refresh_keys.entity_id',
})
.from('refresh_keys');
})
.update({ next_update_at: tx.fn.now() });
}
async transaction<T>(fn: (tx: Transaction) => Promise<T>): Promise<T> {
try {
let result: T | undefined = undefined;
plugins/catalog-backend/src/database/tables.ts
+5
View File
@@ -43,6 +43,11 @@ export type DbRefreshStateRow = {
location_key?: string;
};
export type DbRefreshKeysRow = {
entity_id: string;
key: string;
};
export type DbRefreshStateReferencesRow = {
source_key?: string;
source_entity_ref?: string;
plugins/catalog-backend/src/database/types.ts
+6 -1
View File
@@ -18,7 +18,7 @@ import { Entity } from '@backstage/catalog-model';
import { JsonObject } from '@backstage/types';
import { DateTime } from 'luxon';
import { EntityRelationSpec } from '../api';
import { DeferredEntity } from '../processing/types';
import { DeferredEntity, RefreshKeyData } from '../processing/types';
import { DbRelationsRow } from './tables';
/**
@@ -38,6 +38,7 @@ export type UpdateProcessedEntityOptions = {
relations: EntityRelationSpec[];
deferredEntities: DeferredEntity[];
locationKey?: string;
refreshKeys: RefreshKeyData[];
};
export type UpdateEntityCacheOptions = {
@@ -81,6 +82,10 @@ export type ReplaceUnprocessedEntitiesOptions =
type: 'delta';
};
export type RefreshByKeyOptions = {
keys: string[];
};
export type RefreshOptions = {
entityRef: string;
};
plugins/catalog-backend/src/modules/core/FileReaderProcessor.test.ts
+18 -5
View File
@@ -43,7 +43,10 @@ describe('FileReaderProcessor', () => {
expect(generated.type).toBe('entity');
expect(generated.location).toEqual(spec);
expect(generated.entity).toEqual({ kind: 'Component' });
expect(generated.entity).toEqual({
kind: 'Component',
metadata: { name: 'component-test' },
});
});
it('should fail load from file with error', async () => {
@@ -77,14 +80,24 @@ describe('FileReaderProcessor', () => {
defaultEntityDataParser,
);
expect(emit).toBeCalledTimes(2);
expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' });
expect(emit).toBeCalledTimes(4);
expect(emit.mock.calls[0][0].entity).toEqual({
kind: 'Component',
metadata: { name: 'component-test' },
});
expect(emit.mock.calls[0][0].location).toEqual({
type: 'file',
target: expect.stringMatching(/^[^*]*$/),
});
expect(emit.mock.calls[1][0].entity).toEqual({ kind: 'API' });
expect(emit.mock.calls[1][0].location).toEqual({
expect(emit.mock.calls[1][0].key).toContain('file:');
expect(emit.mock.calls[1][0].key).toContain(
'fileReaderProcessor/component.yaml',
);
expect(emit.mock.calls[2][0].entity).toEqual({
kind: 'API',
metadata: { name: 'api-test' },
});
expect(emit.mock.calls[2][0].location).toEqual({
type: 'file',
target: expect.stringMatching(/^[^*]*$/),
});
plugins/catalog-backend/src/modules/core/FileReaderProcessor.ts
+11 -3
View File
@@ -28,6 +28,8 @@ import {
const glob = promisify(g);
const LOCATION_TYPE = 'file';
/** @public */
export class FileReaderProcessor implements CatalogProcessor {
getProcessorName(): string {
@@ -40,7 +42,7 @@ export class FileReaderProcessor implements CatalogProcessor {
emit: CatalogProcessorEmit,
parser: CatalogProcessorParser,
): Promise<boolean> {
if (location.type !== 'file') {
if (location.type !== LOCATION_TYPE) {
return false;
}
@@ -50,17 +52,23 @@ export class FileReaderProcessor implements CatalogProcessor {
if (fileMatches.length > 0) {
for (const fileMatch of fileMatches) {
const data = await fs.readFile(fileMatch);
const normalizedFilePath = path.normalize(fileMatch);
// The normalize converts to native slashes; the glob library returns
// forward slashes even on windows
for await (const parseResult of parser({
data: data,
location: {
type: 'file',
target: path.normalize(fileMatch),
type: LOCATION_TYPE,
target: normalizedFilePath,
},
})) {
emit(parseResult);
emit(
processingResult.refresh(
`${LOCATION_TYPE}:${normalizedFilePath}`,
),
);
}
}
} else if (!optional) {
plugins/catalog-backend/src/modules/core/PlaceholderProcessor.test.ts
+42 -3
View File
@@ -17,6 +17,7 @@ import { UrlReader } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import { CatalogProcessorResult } from '../../api';
import {
jsonPlaceholderResolver,
PlaceholderProcessor,
@@ -51,7 +52,7 @@ describe('PlaceholderProcessor', () => {
integrations,
});
await expect(
processor.preProcessEntity(input, { type: 't', target: 'l' }),
processor.preProcessEntity(input, { type: 't', target: 'l' }, () => {}),
).resolves.toBe(input);
});
@@ -76,6 +77,7 @@ describe('PlaceholderProcessor', () => {
spec: { a: [{ b: { $upper: 'text' } }] },
},
{ type: 'fake', target: 'http://example.com' },
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -110,7 +112,7 @@ describe('PlaceholderProcessor', () => {
};
await expect(
processor.preProcessEntity(entity, { type: 'a', target: 'b' }),
processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}),
).resolves.toEqual(entity);
expect(read).not.toBeCalled();
@@ -131,7 +133,7 @@ describe('PlaceholderProcessor', () => {
};
await expect(
processor.preProcessEntity(entity, { type: 'a', target: 'b' }),
processor.preProcessEntity(entity, { type: 'a', target: 'b' }, () => {}),
).resolves.toEqual(entity);
expect(read).not.toBeCalled();
@@ -158,6 +160,7 @@ describe('PlaceholderProcessor', () => {
target:
'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
},
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -194,6 +197,7 @@ describe('PlaceholderProcessor', () => {
target:
'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
},
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -228,6 +232,7 @@ describe('PlaceholderProcessor', () => {
target:
'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
},
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -266,6 +271,7 @@ describe('PlaceholderProcessor', () => {
target:
'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
},
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -303,6 +309,7 @@ describe('PlaceholderProcessor', () => {
type: 'url',
target: './a/b/catalog-info.yaml',
},
() => {},
),
).resolves.toEqual({
apiVersion: 'a',
@@ -343,6 +350,7 @@ describe('PlaceholderProcessor', () => {
type: 'url',
target: './a/b/catalog-info.yaml',
},
() => {},
),
).rejects.toThrow(
/^Placeholder \$text could not form a URL out of \.\/a\/b\/catalog-info\.yaml and \.\.\/c\/catalog-info\.yaml, TypeError \[ERR_INVALID_URL\]/,
@@ -350,6 +358,35 @@ describe('PlaceholderProcessor', () => {
expect(read).not.toBeCalled();
});
it('should emit the resolverValue as a refreshKey', async () => {
read.mockResolvedValue(
Buffer.from(JSON.stringify({ a: ['b', 7] }), 'utf-8'),
);
const processor = new PlaceholderProcessor({
resolvers: {
json: jsonPlaceholderResolver,
},
reader,
integrations,
});
const emitted = new Array<CatalogProcessorResult>();
await processor.preProcessEntity(
{
apiVersion: 'a',
kind: 'k',
metadata: { name: 'n' },
spec: { a: [{ b: { $json: './path-to-file.json' } }] },
},
{ type: 'fake', target: 'http://example.com' },
result => emitted.push(result),
);
expect(emitted[0]).toEqual({
type: 'refresh',
key: 'url:http://example.com/path-to-file.json',
});
});
});
describe('yamlPlaceholderResolver', () => {
@@ -360,6 +397,7 @@ describe('yamlPlaceholderResolver', () => {
baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
read,
resolveUrl: (url, base) => integrations.resolveUrl({ url, base }),
emit: () => {},
};
beforeEach(() => {
@@ -405,6 +443,7 @@ describe('jsonPlaceholderResolver', () => {
baseUrl: 'https://github.com/backstage/backstage/a/b/catalog-info.yaml',
read,
resolveUrl: (url, base) => integrations.resolveUrl({ url, base }),
emit: () => {},
};
beforeEach(() => {
plugins/catalog-backend/src/modules/core/PlaceholderProcessor.ts
+25 -8
View File
@@ -19,7 +19,12 @@ import { Entity } from '@backstage/catalog-model';
import { JsonValue } from '@backstage/types';
import { ScmIntegrationRegistry } from '@backstage/integration';
import yaml from 'yaml';
import { CatalogProcessor, LocationSpec } from '../../api';
import {
CatalogProcessor,
CatalogProcessorEmit,
LocationSpec,
processingResult,
} from '../../api';
/** @public */
export type PlaceholderResolverRead = (url: string) => Promise<Buffer>;
@@ -37,6 +42,7 @@ export type PlaceholderResolverParams = {
baseUrl: string;
read: PlaceholderResolverRead;
resolveUrl: PlaceholderResolverResolveUrl;
emit: CatalogProcessorEmit;
};
/** @public */
@@ -66,6 +72,7 @@ export class PlaceholderProcessor implements CatalogProcessor {
async preProcessEntity(
entity: Entity,
location: LocationSpec,
emit: CatalogProcessorEmit,
): Promise<Entity> {
const process = async (data: any): Promise<[any, boolean]> => {
if (!data || !(data instanceof Object)) {
@@ -102,6 +109,7 @@ export class PlaceholderProcessor implements CatalogProcessor {
const resolverKey = keys[0].substr(1);
const resolverValue = data[keys[0]];
const resolver = this.options.resolvers[resolverKey];
if (!resolver || typeof resolverValue !== 'string') {
// If there was no such placeholder resolver or if the value was not a
@@ -134,6 +142,7 @@ export class PlaceholderProcessor implements CatalogProcessor {
baseUrl: location.target,
read,
resolveUrl,
emit,
}),
true,
];
@@ -151,11 +160,13 @@ export class PlaceholderProcessor implements CatalogProcessor {
export async function yamlPlaceholderResolver(
params: PlaceholderResolverParams,
): Promise<JsonValue> {
const text = await readTextLocation(params);
const { content, url } = await readTextLocation(params);
params.emit(processingResult.refresh(`url:${url}`));
let documents: yaml.Document.Parsed[];
try {
documents = yaml.parseAllDocuments(text).filter(d => d);
documents = yaml.parseAllDocuments(content).filter(d => d);
} catch (e) {
throw new Error(
`Placeholder \$${params.key} failed to parse YAML data at ${params.value}, ${e}`,
@@ -182,10 +193,12 @@ export async function yamlPlaceholderResolver(
export async function jsonPlaceholderResolver(
params: PlaceholderResolverParams,
): Promise<JsonValue> {
const text = await readTextLocation(params);
const { content, url } = await readTextLocation(params);
params.emit(processingResult.refresh(`url:${url}`));
try {
return JSON.parse(text);
return JSON.parse(content);
} catch (e) {
throw new Error(
`Placeholder \$${params.key} failed to parse JSON data at ${params.value}, ${e}`,
@@ -196,7 +209,11 @@ export async function jsonPlaceholderResolver(
export async function textPlaceholderResolver(
params: PlaceholderResolverParams,
): Promise<JsonValue> {
return await readTextLocation(params);
const { content, url } = await readTextLocation(params);
params.emit(processingResult.refresh(`url:${url}`));
return content;
}
/*
@@ -205,12 +222,12 @@ export async function textPlaceholderResolver(
async function readTextLocation(
params: PlaceholderResolverParams,
): Promise<string> {
): Promise<{ content: string; url: string }> {
const newUrl = relativeUrl(params);
try {
const data = await params.read(newUrl);
return data.toString('utf-8');
return { content: data.toString('utf-8'), url: newUrl };
} catch (e) {
throw new Error(
`Placeholder \$${params.key} could not read location ${params.value}, ${e}`,
plugins/catalog-backend/src/modules/core/UrlReaderProcessor.test.ts
+20 -4
View File
@@ -61,7 +61,13 @@ describe('UrlReaderProcessor', () => {
server.use(
rest.get(`${mockApiOrigin}/component.yaml`, (_, res, ctx) =>
res(ctx.set({ ETag: 'my-etag' }), ctx.json({ mock: 'entity' })),
res(
ctx.set({ ETag: 'my-etag' }),
ctx.json({
kind: 'component',
metadata: { name: 'mock-url-entity' },
}),
),
),
);
@@ -74,15 +80,25 @@ describe('UrlReaderProcessor', () => {
mockCache,
);
expect(emitted.length).toBe(1);
expect(emitted.length).toBe(2);
expect(emitted[0]).toEqual({
type: 'entity',
location: spec,
entity: { mock: 'entity' },
entity: { kind: 'component', metadata: { name: 'mock-url-entity' } },
});
expect(emitted[1]).toEqual({
type: 'refresh',
key: 'url:http://localhost/component.yaml',
});
expect(mockCache.set).toBeCalledWith('v1', {
etag: 'my-etag',
value: [{ type: 'entity', location: spec, entity: { mock: 'entity' } }],
value: [
{
type: 'entity',
location: spec,
entity: { kind: 'component', metadata: { name: 'mock-url-entity' } },
},
],
});
expect(mockCache.set).toBeCalledTimes(1);
});
plugins/catalog-backend/src/modules/core/UrlReaderProcessor.ts
+2
View File
@@ -93,6 +93,8 @@ export class UrlReaderProcessor implements CatalogProcessor {
value: parseResults as CatalogProcessorEntityResult[],
});
}
emit(processingResult.refresh(`${location.type}:${location.target}`));
} catch (error) {
assertError(error);
const message = `Unable to read ${location.type}, ${error}`;
plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/component.yaml
+2
View File
@@ -1 +1,3 @@
kind: Component
metadata:
name: component-test
plugins/catalog-backend/src/modules/core/__fixtures__/fileReaderProcessor/dir/api.yaml
+2
View File
@@ -1 +1,3 @@
kind: API
metadata:
name: api-test
plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.test.ts
+6
View File
@@ -30,6 +30,7 @@ describe('DefaultCatalogProcessingEngine', () => {
updateProcessedEntity: jest.fn(),
updateEntityCache: jest.fn(),
listParents: jest.fn(),
setRefreshKeys: jest.fn(),
} as unknown as jest.Mocked<DefaultProcessingDatabase>;
const orchestrator: jest.Mocked<CatalogProcessingOrchestrator> = {
process: jest.fn(),
@@ -58,6 +59,7 @@ describe('DefaultCatalogProcessingEngine', () => {
errors: [],
deferredEntities: [],
state: {},
refreshKeys: [],
});
const engine = new DefaultCatalogProcessingEngine(
getVoidLogger(),
@@ -123,6 +125,7 @@ describe('DefaultCatalogProcessingEngine', () => {
errors: [],
deferredEntities: [],
state: {},
refreshKeys: [],
});
const engine = new DefaultCatalogProcessingEngine(
getVoidLogger(),
@@ -203,6 +206,7 @@ describe('DefaultCatalogProcessingEngine', () => {
errors: [],
deferredEntities: [],
state: {},
refreshKeys: [],
});
const engine = new DefaultCatalogProcessingEngine(
@@ -413,6 +417,7 @@ describe('DefaultCatalogProcessingEngine', () => {
errors: [],
deferredEntities: [],
state: {},
refreshKeys: [],
})
.mockResolvedValueOnce({
ok: true,
@@ -432,6 +437,7 @@ describe('DefaultCatalogProcessingEngine', () => {
errors: [],
deferredEntities: [],
state: {},
refreshKeys: [],
});
await engine.start();
plugins/catalog-backend/src/processing/DefaultCatalogProcessingEngine.ts
+2
View File
@@ -134,6 +134,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine {
.update(stableStringify({ ...result.completedEntity }))
.update(stableStringify([...result.deferredEntities]))
.update(stableStringify([...result.relations]))
.update(stableStringify([...result.refreshKeys]))
.update(stableStringify([...parents]));
}
@@ -180,6 +181,7 @@ export class DefaultCatalogProcessingEngine implements CatalogProcessingEngine {
relations: result.relations,
deferredEntities: result.deferredEntities,
locationKey,
refreshKeys: result.refreshKeys,
});
oldRelationSources = new Set(
previous.relations.map(r => r.source_entity_ref),
plugins/catalog-backend/src/processing/DefaultCatalogProcessingOrchestrator.test.ts
+2
View File
@@ -102,6 +102,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => {
ok: true,
completedEntity: entity,
deferredEntities: [],
refreshKeys: [],
errors: [],
relations: [],
state: {
@@ -119,6 +120,7 @@ describe('DefaultCatalogProcessingOrchestrator', () => {
).resolves.toEqual({
ok: true,
completedEntity: entity,
refreshKeys: [],
deferredEntities: [
{
locationKey: 'url:./new-place',
plugins/catalog-backend/src/processing/ProcessorOutputCollector.ts
+5 -1
View File
@@ -24,7 +24,7 @@ import { assertError } from '@backstage/errors';
import { Logger } from 'winston';
import { CatalogProcessorResult, EntityRelationSpec } from '../api';
import { locationSpecToLocationEntity } from '../util/conversion';
import { DeferredEntity } from './types';
import { DeferredEntity, RefreshKeyData } from './types';
import {
getEntityLocationRef,
getEntityOriginLocationRef,
@@ -38,6 +38,7 @@ export class ProcessorOutputCollector {
private readonly errors = new Array<Error>();
private readonly relations = new Array<EntityRelationSpec>();
private readonly deferredEntities = new Array<DeferredEntity>();
private readonly refreshKeys = new Array<RefreshKeyData>();
private done = false;
constructor(
@@ -54,6 +55,7 @@ export class ProcessorOutputCollector {
return {
errors: this.errors,
relations: this.relations,
refreshKeys: this.refreshKeys,
deferredEntities: this.deferredEntities,
};
}
@@ -116,6 +118,8 @@ export class ProcessorOutputCollector {
this.relations.push(i.relation);
} else if (i.type === 'error') {
this.errors.push(i.error);
} else if (i.type === 'refresh') {
this.refreshKeys.push({ key: i.key });
}
}
}
plugins/catalog-backend/src/processing/types.ts
+9 -1
View File
@@ -26,7 +26,6 @@ export type EntityProcessingRequest = {
entity: Entity;
state?: JsonObject; // Versions for multiple deployments etc
};
/**
* The result of processing an entity.
* @public
@@ -38,6 +37,7 @@ export type EntityProcessingResult =
completedEntity: Entity;
deferredEntities: DeferredEntity[];
relations: EntityRelationSpec[];
refreshKeys: RefreshKeyData[];
errors: Error[];
}
| {
@@ -45,6 +45,14 @@ export type EntityProcessingResult =
errors: Error[];
};
/**
* A string to associate to the entity itself.
* @public
*/
export type RefreshKeyData = {
key: string;
};
/**
* Responsible for executing the individual processing steps in order to fully process an entity.
* @public
plugins/catalog-backend/src/service/DefaultLocationService.test.ts
+6
View File
@@ -48,6 +48,7 @@ describe('DefaultLocationServiceTest', () => {
name: 'foo',
},
},
refreshKeys: [],
deferredEntities: [
{
entity: {
@@ -75,6 +76,7 @@ describe('DefaultLocationServiceTest', () => {
},
},
deferredEntities: [],
refreshKeys: [],
relations: [],
errors: [],
});
@@ -134,6 +136,7 @@ describe('DefaultLocationServiceTest', () => {
},
},
deferredEntities: [],
refreshKeys: [],
relations: [],
errors: [],
});
@@ -161,6 +164,7 @@ describe('DefaultLocationServiceTest', () => {
name: 'foo',
},
},
refreshKeys: [],
deferredEntities: [
{
entity: {
@@ -188,6 +192,7 @@ describe('DefaultLocationServiceTest', () => {
},
},
deferredEntities: [],
refreshKeys: [],
relations: [],
errors: [],
});
@@ -211,6 +216,7 @@ describe('DefaultLocationServiceTest', () => {
name: 'bar',
},
},
refreshKeys: [],
deferredEntities: [],
relations: [],
errors: [],
plugins/catalog-backend/src/service/DefaultRefreshService.test.ts
+1
View File
@@ -138,6 +138,7 @@ describe('Refresh integration', () => {
errors: [],
deferredEntities,
state: {},
refreshKeys: [],
};
},
},
plugins/catalog-backend/src/service/createRouter.test.ts
+1
View File
@@ -409,6 +409,7 @@ describe('createRouter readonly disabled', () => {
state: {},
completedEntity: entity,
deferredEntities: [],
refreshKeys: [],
relations: [],
errors: [],
});