paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

Tweaks to the v2 catalog ingestion

Browse Source 
Signed-off-by: Fredrik Adelöw <freben@gmail.com>
This commit is contained in:
Fredrik Adelöw
2021-04-28 17:05:55 +02:00
parent c7de293327
commit d6cfbc797f
6 changed files with 50 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/migrationsv2/20210302150147_refresh_state.js
+11 -3
View File
@@ -83,10 +83,18 @@ exports.up = async function up(knex) {
.inTable('refresh_state')
.onDelete('CASCADE')
.comment(
'Entity ID which correspond to the ID in the refresh_state table',
'Entity ID which corresponds to the ID in the refresh_state table',
);
table.text('etag').notNullable().comment('Etag to be used for caching');
table.text('finalized_entity').notNullable().comment('The final entity');
table
.text('hash')
.notNullable()
.comment(
'Stable hash of the entity data, to be used for caching and avoiding redundant work',
);
table
.text('final_entity')
.notNullable()
.comment('The JSON encoded final entity');
table.index('entity_id', 'final_entities_entity_id_idx');
});
plugins/catalog-backend/src/next/NextEntitiesCatalog.ts
+1 -1
View File
@@ -37,7 +37,7 @@ export class NextEntitiesCatalog implements EntitiesCatalog {
'final_entities',
).select();
const entities = dbResponse.map(e => JSON.parse(e.finalized_entity));
const entities = dbResponse.map(e => JSON.parse(e.final_entity));
return {
entities,
plugins/catalog-backend/src/next/Stitcher.test.ts
+13 -12
View File
@@ -74,12 +74,12 @@ describe('Stitcher', () => {
await stitcher.stitch(new Set(['k:ns/n']));
let firstEtag: string;
let firstHash: string;
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].finalized_entity);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: [
{
@@ -105,12 +105,13 @@ describe('Stitcher', () => {
},
});
firstEtag = entity.metadata.etag;
expect(entity.metadata.etag).toEqual(entities[0].hash);
firstHash = entities[0].hash;
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
@@ -127,9 +128,9 @@ describe('Stitcher', () => {
await db.transaction(async tx => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].finalized_entity);
expect(entities[0].etag).toEqual(firstEtag);
expect(entity.metadata.etag).toEqual(firstEtag);
const entity = JSON.parse(entities[0].final_entity);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
});
// Now add one more relation and re-stitch
@@ -150,7 +151,7 @@ describe('Stitcher', () => {
const entities = await tx<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
const entity = JSON.parse(entities[0].finalized_entity);
const entity = JSON.parse(entities[0].final_entity);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
@@ -184,14 +185,14 @@ describe('Stitcher', () => {
},
});
expect(entities[0].etag).not.toEqual(firstEtag);
expect(entities[0].etag).toEqual(entity.metadata.etag);
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
const search = await tx<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/other' },
{ entity_id: 'my-id', key: 'relations', value: 'looksat:k:ns/third' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
plugins/catalog-backend/src/next/Stitcher.ts
+21 -15
View File
@@ -31,11 +31,11 @@ const BATCH_SIZE = 50;
export type DbFinalEntitiesRow = {
entity_id: string;
etag: string;
finalized_entity: string;
hash: string;
final_entity: string;
};
function generateEntityEtag(entity: Entity) {
function generateStableHash(entity: Entity) {
return createHash('sha1')
.update(stableStringify({ ...entity }))
.digest('hex');
@@ -64,7 +64,7 @@ export class Stitcher {
processedEntity?: string;
errors: string;
incomingReferenceCount: string | number;
previousEtag?: string;
previousHash?: string;
relationType?: string;
relationTarget?: string;
}> = await tx
@@ -79,7 +79,7 @@ export class Stitcher {
processedEntity: 'refresh_state.processed_entity',
errors: 'refresh_state.errors',
incomingReferenceCount: 'incoming_references.count',
previousEtag: 'final_entities.etag',
previousHash: 'final_entities.hash',
relationType: 'relations.type',
relationTarget: 'relations.target_entity_ref',
})
@@ -111,7 +111,7 @@ export class Stitcher {
processedEntity,
// errors,
incomingReferenceCount,
previousEtag,
previousHash,
} = result[0];
// If there was no processed entity in place, the target hasn't been
@@ -125,7 +125,8 @@ export class Stitcher {
return;
}
// Grab the processed entity and stitch all of the relevant data into it
// Grab the processed entity and stitch all of the relevant data into
// it
const entity = JSON.parse(processedEntity) as Entity;
const isOrphan = Number(incomingReferenceCount) === 0;
@@ -137,33 +138,38 @@ export class Stitcher {
};
}
// TODO: entityRef is lower case and should be uppercase in the final result
// TODO: entityRef is lower case and should be uppercase in the final
// result
entity.relations = result
.filter(row => row.relationType)
.filter(row => row.relationType /* exclude null row, if relevant */)
.map(row => ({
type: row.relationType!,
target: parseEntityRef(row.relationTarget!),
}));
// If the output entity was actually not changed, just abort
const etag = generateEntityEtag(entity);
if (etag === previousEtag) {
const hash = generateStableHash(entity);
if (hash === previousHash) {
this.logger.debug(`Skipped stitching of ${entityRef}, no changes`);
return;
}
entity.metadata.uid = entityId;
entity.metadata.generation = 1;
entity.metadata.etag = etag;
if (!entity.metadata.etag) {
// If the original data source did not have its own etag handling,
// use the hash as a good-quality etag
entity.metadata.etag = hash;
}
await tx<DbFinalEntitiesRow>('final_entities')
.insert({
entity_id: entityId,
finalized_entity: JSON.stringify(entity),
etag,
final_entity: JSON.stringify(entity),
hash,
})
.onConflict('entity_id')
.merge(['finalized_entity', 'etag']);
.merge(['final_entity', 'hash']);
const searchEntries = buildEntitySearch(entityId, entity);
await tx<DbSearchRow>('search').where({ entity_id: entityId }).delete();
plugins/catalog-backend/src/next/search.test.ts
+2 -2
View File
@@ -152,8 +152,8 @@ describe('search', () => {
key: 'metadata.namespace',
value: ENTITY_DEFAULT_NAMESPACE,
},
{ entity_id: 'eid', key: 'relations', value: 't1:k:ns/a' },
{ entity_id: 'eid', key: 'relations', value: 't2:k:ns/b' },
{ entity_id: 'eid', key: 'relations.t1', value: 'k:ns/a' },
{ entity_id: 'eid', key: 'relations.t2', value: 'k:ns/b' },
]);
});
});
plugins/catalog-backend/src/next/search.ts
+2 -2
View File
@@ -182,8 +182,8 @@ export function buildEntitySearch(
// Visit relations
for (const relation of entity.relations ?? []) {
raw.push({
key: 'relations',
value: `${relation.type}:${stringifyEntityRef(relation.target)}`,
key: `relations.${relation.type}`,
value: stringifyEntityRef(relation.target),
});
}