diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 742883e784..abe581fbc3 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index 9afd32b774..d12d5bec3e 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -16,7 +16,10 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { - table.timestamp('last_updated_at').nullable(); + table + .bigint('last_updated_at') + .nullable() + .comment('The time when final_entity changed'); }); }; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index e3f3efd1bb..2778cf2fba 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at?: string | Date; + last_updated_at: string | null; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 1a0fe98b40..a305df52e3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -213,7 +213,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { let entities: Entity[] = rows.map(e => { const entityJson = JSON.parse(e.final_entity!); if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = e.last_updated_at; } return entityJson; @@ -272,7 +272,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const row of await query) { const entityJson = JSON.parse(row.entity); if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = row.entity.last_updated_at; } lookup.set(row.entityRef, row.entity ? entityJson : null); @@ -393,7 +393,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } const entityJson = JSON.parse(rootRow.entityJson); if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = rootRow.last_updated_at; } const rootEntity = entityJson as Entity; diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a3657e45fd..a13467e9bd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,6 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; +import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -42,6 +43,7 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; + const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -110,6 +112,15 @@ describe('Stitcher', () => { }); expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at = entities[0].last_updated_at; + expect(last_updated_at).not.toBeNull(); + const lastUpdatedAt = DateTime.fromMillis( + last_updated_at ? +last_updated_at : 0, + ); + const msAfterStitch = lastUpdatedAt + .diff(timeBeforeStitch, 'milliseconds') + .toObject(); + expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -127,7 +138,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', @@ -174,6 +190,7 @@ describe('Stitcher', () => { }, ]); + const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -206,6 +223,15 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); + expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at_after_restitch = entities[0].last_updated_at; + expect(last_updated_at_after_restitch).not.toBeNull(); + const msAfterRestitch = DateTime.fromMillis( + last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, + ) + .diff(timeBeforeRestitch, 'milliseconds') + .toObject(); + expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ @@ -227,7 +253,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 64f5527262..c3892244dd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,6 +23,7 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -207,7 +208,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: this.database.fn.now(), + last_updated_at: `${DateTime.now().toMillis()}`, }) .where('entity_id', entityId) .where('stitch_ticket', ticket)