Merge pull request #14980 from iriswang233/Track-the-last-time-the-final-entity-changed

Track the last time the final entity changed

.changeset/lucky-singers-worry.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Track the last time the final entity changed with new timestamp "last updated at" data in final entities database, which gets updated with the time when final entity is updated.

plugins/catalog-backend/migrations/20221201085245_add_last_updated_at_in_final_entities.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-check
+
+/**
+ * @param { import("knex").Knex } knex
+ */
+exports.up = async function up(knex) {
+  await knex.schema.table('final_entities', table => {
+    table
+      .dateTime('last_updated_at')
+      .nullable()
+      .comment('The time when final_entity changed');
+  });
+};
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+exports.down = async function down(knex) {
+  await knex.schema.table('final_entities', table => {
+    table.dropColumn('last_updated_at');
+  });
+};

plugins/catalog-backend/src/database/tables.ts

@@ -66,6 +66,7 @@ export type DbFinalEntitiesRow = {
   hash: string;
   stitch_ticket: string;
   final_entity?: string;
+  last_updated_at?: string | Date;
 };
 
 export type DbSearchRow = {

plugins/catalog-backend/src/migrations.test.ts

@@ -100,4 +100,82 @@ describe('migrations', () => {
     },
     60_000,
   );
+
+  it.each(databases.eachSupportedId())(
+    '20221201085245_add_last_updated_at_in_final_entities.js, %p',
+    async databaseId => {
+      const knex = await databases.init(databaseId);
+
+      await migrateUntilBefore(
+        knex,
+        '20221201085245_add_last_updated_at_in_final_entities.js',
+      );
+
+      await knex
+        .insert([
+          {
+            entity_id: 'my-id',
+            entity_ref: 'k:ns/n',
+            unprocessed_entity: JSON.stringify({}),
+            processed_entity: JSON.stringify({
+              apiVersion: 'a',
+              kind: 'k',
+              metadata: {
+                name: 'n',
+                namespace: 'ns',
+              },
+              spec: {
+                k: 'v',
+              },
+            }),
+            errors: '[]',
+            next_update_at: knex.fn.now(),
+            last_discovery_at: knex.fn.now(),
+          },
+        ])
+        .into('refresh_state');
+
+      await knex
+        .insert({
+          entity_id: 'my-id',
+          hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e',
+          stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312',
+          final_entity:
+            '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}',
+        })
+        .into('final_entities');
+
+      await migrateUpOnce(knex);
+
+      await expect(knex('final_entities')).resolves.toEqual(
+        expect.arrayContaining([
+          {
+            entity_id: 'my-id',
+            hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e',
+            stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312',
+            final_entity:
+              '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}',
+            last_updated_at: null,
+          },
+        ]),
+      );
+
+      await migrateDownOnce(knex);
+
+      await expect(knex('final_entities')).resolves.toEqual(
+        expect.arrayContaining([
+          {
+            entity_id: 'my-id',
+            hash: 'd1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e',
+            stitch_ticket: '52367ed7-120b-405f-b7e0-cdd90f956312',
+            final_entity:
+              '{"apiVersion":"a","kind":"k","metadata":{"name":"n","namespace":"ns","uid":"my-id","etag":"d1c0c56d5fea4238e4c091d9dde4cb42d05a6b7e"},"spec":{"k":"v"},"relations":[{"type":"looksAt","targetRef":"k:ns/other"}]}',
+          },
+        ]),
+      );
+
+      await knex.destroy();
+    },
+    60_000,
+  );
 });

plugins/catalog-backend/src/stitching/Stitcher.test.ts

@@ -110,6 +110,8 @@ describe('Stitcher', () => {
       });
 
       expect(entity.metadata.etag).toEqual(entities[0].hash);
+      const last_updated_at = entities[0].last_updated_at;
+      expect(last_updated_at).not.toBeNull();
       const firstHash = entities[0].hash;
 
       const search = await db<DbSearchRow>('search');
@@ -127,7 +129,12 @@ describe('Stitcher', () => {
             original_value: 'a',
             value: 'a',
           },
-          { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' },
+          {
+            entity_id: 'my-id',
+            key: 'kind',
+            original_value: 'k',
+            value: 'k',
+          },
           {
             entity_id: 'my-id',
             key: 'metadata.name',
@@ -227,7 +234,12 @@ describe('Stitcher', () => {
             original_value: 'a',
             value: 'a',
           },
-          { entity_id: 'my-id', key: 'kind', original_value: 'k', value: 'k' },
+          {
+            entity_id: 'my-id',
+            key: 'kind',
+            original_value: 'k',
+            value: 'k',
+          },
           {
             entity_id: 'my-id',
             key: 'metadata.name',

plugins/catalog-backend/src/stitching/Stitcher.ts

@@ -207,11 +207,12 @@ export class Stitcher {
       .update({
         final_entity: JSON.stringify(entity),
         hash,
+        last_updated_at: this.database.fn.now(),
       })
       .where('entity_id', entityId)
       .where('stitch_ticket', ticket)
       .onConflict('entity_id')
-      .merge(['final_entity', 'hash']);
+      .merge(['final_entity', 'hash', 'last_updated_at']);
 
     if (amountOfRowsChanged === 0) {
       this.logger.debug(