From facbae6feff8b6715b75722e499f4fe876a8d6f7 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:00:32 +0800 Subject: [PATCH 01/13] Add last_updated_at in final_entities table Signed-off-by: iris --- ...5_add_last_updated_at_in_final_entities.js | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js new file mode 100644 index 0000000000..9afd32b774 --- /dev/null +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -0,0 +1,31 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.table('final_entities', table => { + table.timestamp('last_updated_at').nullable(); + }); +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + await knex.schema.table('final_entities', table => { + table.dropColumn('last_updated_at'); + }); +}; From 253acc8b4395b2bdac6afdf651aba3b8c15bdc5f Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 1 Dec 2022 17:08:36 +0800 Subject: [PATCH 02/13] update last_updated_at when update final_entities Signed-off-by: iris --- plugins/catalog-backend/src/database/tables.ts | 1 + plugins/catalog-backend/src/stitching/Stitcher.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index c23753b267..e3f3efd1bb 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,6 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; + last_updated_at?: string | Date; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 0da8e39694..64f5527262 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -207,6 +207,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, + last_updated_at: this.database.fn.now(), }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From a483d5f5a4d82219e4e41e764127020de02ea9c9 Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 12:05:18 +0800 Subject: [PATCH 03/13] add last_updated_at as annotation backstage.io/last_updated-at Signed-off-by: iris --- .../src/service/DefaultEntitiesCatalog.ts | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 14f22f862b..1a0fe98b40 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -210,7 +210,14 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); + let entities: Entity[] = rows.map(e => { + const entityJson = JSON.parse(e.final_entity!); + if (e.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + e.last_updated_at; + } + return entityJson; + }); if (request?.fields) { entities = entities.map(e => request.fields!(e)); @@ -263,7 +270,12 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query = parseFilter(request.filter, query, this.database); } for (const row of await query) { - lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); + const entityJson = JSON.parse(row.entity); + if (row.entity.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + row.entity.last_updated_at; + } + lookup.set(row.entityRef, row.entity ? entityJson : null); } } @@ -373,13 +385,18 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .where('refresh_state.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', + last_updated_at: 'final_entities.last_updated_at', }); if (!rootRow) { throw new NotFoundError(`No such entity ${rootRef}`); } - - const rootEntity = JSON.parse(rootRow.entityJson) as Entity; + const entityJson = JSON.parse(rootRow.entityJson); + if (rootRow.last_updated_at) { + entityJson.metadata.annotations['backstage.io/last_updated-at'] = + rootRow.last_updated_at; + } + const rootEntity = entityJson as Entity; const seenEntityRefs = new Set(); const todo = new Array(); const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); From 93870e4df1a1a7d9f5f09fed5c333bcf5f97735f Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:00:31 +0800 Subject: [PATCH 04/13] Add changeset Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-singers-worry.md diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md new file mode 100644 index 0000000000..742883e784 --- /dev/null +++ b/.changeset/lucky-singers-worry.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': major +--- + +Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. From 68cf4f593416e71248c4e38f9654bfccc351f83c Mon Sep 17 00:00:00 2001 From: iris Date: Fri, 2 Dec 2022 13:02:43 +0800 Subject: [PATCH 05/13] fix testing Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- ...5_add_last_updated_at_in_final_entities.js | 5 ++- .../catalog-backend/src/database/tables.ts | 2 +- .../src/service/DefaultEntitiesCatalog.ts | 6 ++-- .../src/stitching/Stitcher.test.ts | 35 +++++++++++++++++-- .../catalog-backend/src/stitching/Stitcher.ts | 3 +- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 742883e784..abe581fbc3 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last_updated_at" data in final_entities database, which gets updated with the time when final_entities is updated. And it's displayed in metadata annotation as backstage.io/last_updated-at. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index 9afd32b774..d12d5bec3e 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -16,7 +16,10 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { - table.timestamp('last_updated_at').nullable(); + table + .bigint('last_updated_at') + .nullable() + .comment('The time when final_entity changed'); }); }; diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index e3f3efd1bb..2778cf2fba 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at?: string | Date; + last_updated_at: string | null; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index 1a0fe98b40..a305df52e3 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -213,7 +213,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { let entities: Entity[] = rows.map(e => { const entityJson = JSON.parse(e.final_entity!); if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = e.last_updated_at; } return entityJson; @@ -272,7 +272,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { for (const row of await query) { const entityJson = JSON.parse(row.entity); if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = row.entity.last_updated_at; } lookup.set(row.entityRef, row.entity ? entityJson : null); @@ -393,7 +393,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { } const entityJson = JSON.parse(rootRow.entityJson); if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated-at'] = + entityJson.metadata.annotations['backstage.io/last_updated_at'] = rootRow.last_updated_at; } const rootEntity = entityJson as Entity; diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a3657e45fd..a13467e9bd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,6 +17,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; +import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -42,6 +43,7 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; + const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -110,6 +112,15 @@ describe('Stitcher', () => { }); expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at = entities[0].last_updated_at; + expect(last_updated_at).not.toBeNull(); + const lastUpdatedAt = DateTime.fromMillis( + last_updated_at ? +last_updated_at : 0, + ); + const msAfterStitch = lastUpdatedAt + .diff(timeBeforeStitch, 'milliseconds') + .toObject(); + expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -127,7 +138,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', @@ -174,6 +190,7 @@ describe('Stitcher', () => { }, ]); + const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -206,6 +223,15 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); + expect(entity.metadata.etag).toEqual(entities[0].hash); + const last_updated_at_after_restitch = entities[0].last_updated_at; + expect(last_updated_at_after_restitch).not.toBeNull(); + const msAfterRestitch = DateTime.fromMillis( + last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, + ) + .diff(timeBeforeRestitch, 'milliseconds') + .toObject(); + expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ @@ -227,7 +253,12 @@ describe('Stitcher', () => { original_value: 'a', value: 'a', }, - { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' }, + { + entity_id: 'my-id', + key: 'kind', + original_value: 'k', + value: 'k', + }, { entity_id: 'my-id', key: 'metadata.name', diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index 64f5527262..c3892244dd 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,6 +23,7 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; +import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -207,7 +208,7 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: this.database.fn.now(), + last_updated_at: `${DateTime.now().toMillis()}`, }) .where('entity_id', entityId) .where('stitch_ticket', ticket) From 7da60b5b4f5c9df18b524ad2d0957121d2b3cbed Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 12:36:16 +0800 Subject: [PATCH 06/13] revert changes about adding backstage.io/last_updated_at Signed-off-by: iris --- .../src/service/DefaultEntitiesCatalog.ts | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts index a305df52e3..14f22f862b 100644 --- a/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts +++ b/plugins/catalog-backend/src/service/DefaultEntitiesCatalog.ts @@ -210,14 +210,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { }; } - let entities: Entity[] = rows.map(e => { - const entityJson = JSON.parse(e.final_entity!); - if (e.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - e.last_updated_at; - } - return entityJson; - }); + let entities: Entity[] = rows.map(e => JSON.parse(e.final_entity!)); if (request?.fields) { entities = entities.map(e => request.fields!(e)); @@ -270,12 +263,7 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { query = parseFilter(request.filter, query, this.database); } for (const row of await query) { - const entityJson = JSON.parse(row.entity); - if (row.entity.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - row.entity.last_updated_at; - } - lookup.set(row.entityRef, row.entity ? entityJson : null); + lookup.set(row.entityRef, row.entity ? JSON.parse(row.entity) : null); } } @@ -385,18 +373,13 @@ export class DefaultEntitiesCatalog implements EntitiesCatalog { .where('refresh_state.entity_ref', '=', rootRef) .select({ entityJson: 'final_entities.final_entity', - last_updated_at: 'final_entities.last_updated_at', }); if (!rootRow) { throw new NotFoundError(`No such entity ${rootRef}`); } - const entityJson = JSON.parse(rootRow.entityJson); - if (rootRow.last_updated_at) { - entityJson.metadata.annotations['backstage.io/last_updated_at'] = - rootRow.last_updated_at; - } - const rootEntity = entityJson as Entity; + + const rootEntity = JSON.parse(rootRow.entityJson) as Entity; const seenEntityRefs = new Set(); const todo = new Array(); const items = new Array<{ entity: Entity; parentEntityRefs: string[] }>(); From 1003f52b19a952cb5c298f7c39cfb8ba2bfeb3d4 Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 15:05:43 +0800 Subject: [PATCH 07/13] Change last_updated_at to dateTime Signed-off-by: iris --- .../20221201085245_add_last_updated_at_in_final_entities.js | 2 +- plugins/catalog-backend/src/database/tables.ts | 2 +- plugins/catalog-backend/src/stitching/Stitcher.ts | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index d12d5bec3e..e34d45ade6 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -17,7 +17,7 @@ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { table - .bigint('last_updated_at') + .dateTime('last_updated_at') .nullable() .comment('The time when final_entity changed'); }); diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 2778cf2fba..9d89b17d16 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at: string | null; + last_updated_at: string | Date; }; export type DbSearchRow = { diff --git a/plugins/catalog-backend/src/stitching/Stitcher.ts b/plugins/catalog-backend/src/stitching/Stitcher.ts index c3892244dd..4742c81933 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.ts @@ -23,7 +23,6 @@ import { import { SerializedError, stringifyError } from '@backstage/errors'; import { Knex } from 'knex'; import { v4 as uuid } from 'uuid'; -import { DateTime } from 'luxon'; import { Logger } from 'winston'; import { DbFinalEntitiesRow, @@ -208,12 +207,12 @@ export class Stitcher { .update({ final_entity: JSON.stringify(entity), hash, - last_updated_at: `${DateTime.now().toMillis()}`, + last_updated_at: this.database.fn.now(), }) .where('entity_id', entityId) .where('stitch_ticket', ticket) .onConflict('entity_id') - .merge(['final_entity', 'hash']); + .merge(['final_entity', 'hash', 'last_updated_at']); if (amountOfRowsChanged === 0) { this.logger.debug( From 9af905d5d4be8700f4476793cfc2d50b457effbf Mon Sep 17 00:00:00 2001 From: iris Date: Mon, 12 Dec 2022 15:09:48 +0800 Subject: [PATCH 08/13] update test case Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- .../src/stitching/Stitcher.test.ts | 18 ------------------ 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index abe581fbc3..4707a57525 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': major --- -Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. And it's displayed in metadata annotation. +Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index a13467e9bd..31a8e04392 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -17,7 +17,6 @@ import { getVoidLogger } from '@backstage/backend-common'; import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; -import { DateTime } from 'luxon'; import { applyDatabaseMigrations } from '../database/migrations'; import { DbFinalEntitiesRow, @@ -43,7 +42,6 @@ describe('Stitcher', () => { const stitcher = new Stitcher(db, logger); let entities: DbFinalEntitiesRow[]; let entity: Entity; - const timeBeforeStitch = DateTime.now(); await db('refresh_state').insert([ { @@ -114,13 +112,6 @@ describe('Stitcher', () => { expect(entity.metadata.etag).toEqual(entities[0].hash); const last_updated_at = entities[0].last_updated_at; expect(last_updated_at).not.toBeNull(); - const lastUpdatedAt = DateTime.fromMillis( - last_updated_at ? +last_updated_at : 0, - ); - const msAfterStitch = lastUpdatedAt - .diff(timeBeforeStitch, 'milliseconds') - .toObject(); - expect(msAfterStitch.milliseconds).toBeGreaterThan(0); const firstHash = entities[0].hash; const search = await db('search'); @@ -190,7 +181,6 @@ describe('Stitcher', () => { }, ]); - const timeBeforeRestitch = DateTime.now(); await stitcher.stitch(new Set(['k:ns/n'])); entities = await db('final_entities'); @@ -224,14 +214,6 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); expect(entity.metadata.etag).toEqual(entities[0].hash); - const last_updated_at_after_restitch = entities[0].last_updated_at; - expect(last_updated_at_after_restitch).not.toBeNull(); - const msAfterRestitch = DateTime.fromMillis( - last_updated_at_after_restitch ? +last_updated_at_after_restitch : 0, - ) - .diff(timeBeforeRestitch, 'milliseconds') - .toObject(); - expect(msAfterRestitch.milliseconds).toBeGreaterThan(0); expect(await db('search')).toEqual( expect.arrayContaining([ From e2215bc070b5b1256b73492df3ad69c55242f9e3 Mon Sep 17 00:00:00 2001 From: iris Date: Tue, 13 Dec 2022 10:48:00 +0800 Subject: [PATCH 09/13] update changeset to patch Signed-off-by: iris --- .changeset/lucky-singers-worry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/lucky-singers-worry.md b/.changeset/lucky-singers-worry.md index 4707a57525..abefe2bea2 100644 --- a/.changeset/lucky-singers-worry.md +++ b/.changeset/lucky-singers-worry.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': major +'@backstage/plugin-catalog-backend': patch --- Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated. From 24b74c35792742327763e2e1513b4df643ee155d Mon Sep 17 00:00:00 2001 From: iris Date: Tue, 13 Dec 2022 10:51:46 +0800 Subject: [PATCH 10/13] Remove duplicate check Signed-off-by: iris --- plugins/catalog-backend/src/stitching/Stitcher.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-backend/src/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/stitching/Stitcher.test.ts index 31a8e04392..729793a8d7 100644 --- a/plugins/catalog-backend/src/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/stitching/Stitcher.test.ts @@ -213,7 +213,6 @@ describe('Stitcher', () => { expect(entities[0].hash).not.toEqual(firstHash); expect(entities[0].hash).toEqual(entity.metadata.etag); - expect(entity.metadata.etag).toEqual(entities[0].hash); expect(await db('search')).toEqual( expect.arrayContaining([ From bcd800bb33b8a18eb2e587adfabc7daa1e73d2b6 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:28:56 +0800 Subject: [PATCH 11/13] make last_updated_at optional Signed-off-by: iris --- plugins/catalog-backend/src/database/tables.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/database/tables.ts b/plugins/catalog-backend/src/database/tables.ts index 9d89b17d16..e3f3efd1bb 100644 --- a/plugins/catalog-backend/src/database/tables.ts +++ b/plugins/catalog-backend/src/database/tables.ts @@ -66,7 +66,7 @@ export type DbFinalEntitiesRow = { hash: string; stitch_ticket: string; final_entity?: string; - last_updated_at: string | Date; + last_updated_at?: string | Date; }; export type DbSearchRow = { From 860fbcde3b9d154a19411d638e339c47be8086b7 Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:30:52 +0800 Subject: [PATCH 12/13] add ts-check Signed-off-by: iris --- .../20221201085245_add_last_updated_at_in_final_entities.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js index e34d45ade6..44fcf30ba4 100644 --- a/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js +++ b/plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js @@ -14,6 +14,11 @@ * limitations under the License. */ +// @ts-check + +/** + * @param { import("knex").Knex } knex + */ exports.up = async function up(knex) { await knex.schema.table('final_entities', table => { table From b568378cf851ca29467685b797b73b15539c772c Mon Sep 17 00:00:00 2001 From: iris Date: Thu, 15 Dec 2022 15:44:09 +0800 Subject: [PATCH 13/13] add migration testing for 20221201085245_add_last_updated_at_in_final_entities Signed-off-by: iris --- .../catalog-backend/src/migrations.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/plugins/catalog-backend/src/migrations.test.ts b/plugins/catalog-backend/src/migrations.test.ts index 9d9b02fa82..f1ed40dbbe 100644 --- a/plugins/catalog-backend/src/migrations.test.ts +++ b/plugins/catalog-backend/src/migrations.test.ts @@ -100,4 +100,82 @@ describe('migrations', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + '20221201085245_add_last_updated_at_in_final_entities.js, %p', + async databaseId => { + const knex = await databases.init(databaseId); + + await migrateUntilBefore( + knex, + '20221201085245_add_last_updated_at_in_final_entities.js', + ); + + await knex + .insert([ + { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + }, + ]) + .into('refresh_state'); + + await knex + .insert({ + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }) + .into('final_entities'); + + await migrateUpOnce(knex); + + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + last_updated_at: null, + }, + ]), + ); + + await migrateDownOnce(knex); + + await expect(knex('final_entities')).resolves.toEqual( + expect.arrayContaining([ + { + entity_id: 'my-id', + hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e', + stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312', + final_entity: + '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}', + }, + ]), + ); + + await knex.destroy(); + }, + 60_000, + ); });