From e4e6ae88b1862de8ac6a2d2ae66718a7072cd15d Mon Sep 17 00:00:00 2001
From: Johan Haals <johan.haals@gmail.com>
Date: Fri, 16 Apr 2021 09:26:30 +0200
Subject: [PATCH] Add tables for references and relations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Fredrik Adelöw <freben@users.noreply.github.com>
Co-authored-by: Patrik Oldsberg <poldsberg@gmail.com>
Signed-off-by: Johan Haals <johan.haals@gmail.com>
---
 .../20210302150147_refresh_state.js           | 112 +++++++++++++++++-
 1 file changed, 106 insertions(+), 6 deletions(-)

diff --git a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
index 6ee10531eb..a09d892146 100644
--- a/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
+++ b/plugins/catalog-backend/migrations/20210302150147_refresh_state.js
@@ -20,9 +20,63 @@
  * @param {import('knex').Knex} knex
  */
 exports.up = async function up(knex) {
+  await knex.schema.createTable('relations', table => {
+    table.comment('All relations between entities in the catalog');
+    table
+      .text('originating_entity_id')
+      .references('entity_id')
+      .inTable('refresh_state')
+      .onDelete('CASCADE')
+      .notNullable()
+      .comment('The entity that provided the relation');
+    table
+      .text('source_entity_ref')
+      .notNullable()
+      .comment('The entity reference of the source entity of the relation');
+    table
+      .text('type')
+      .notNullable()
+      .comment('The type of the relation between the entities');
+    table
+      .text('target_entity_ref')
+      .notNullable()
+      .comment('The entity reference of the target entity of the relation');
+
+    table.index(
+      ['source_entity_ref', 'type', 'target_entity_ref'],
+      'relations_full_ref_idx',
+    );
+  });
+
+  await knex.schema.createTable('final_entities', table => {
+    table.comment(
+      'This table contains the final entity result after processing and stitching',
+    );
+    table
+      .text('entity_id')
+      .primary()
+      .notNullable()
+      .references('entity_id')
+      .inTable('refresh_state')
+      .onDelete('CASCADE')
+      .comment(
+        'Entity ID which correspond to the ID in the refresh_state table',
+      );
+
+    table.text('finalized_entity').notNullable().comment('The final entity');
+  });
+
   await knex.schema.createTable('refresh_state', table => {
-    table.comment('Location Refresh states');
-    table.text('id').primary().notNullable().comment('Primary id');
+    table.comment(
+      'Location refresh states. Every individual location (that was ever directly or indirectly discovered) and entity has an entry in this table. It therefore represents the entire live set of things that the refresh loop considers.',
+    );
+    table
+      .text('entity_id')
+      .primary()
+      .notNullable()
+      .comment(
+        'Primary ID, which will also be used as the uid of the resulting entity',
+      );
     table
       .text('entity_ref')
       .unique()
@@ -58,8 +112,48 @@ exports.up = async function up(knex) {
       .dateTime('last_discovery_at')
       .notNullable()
       .comment('The last timestamp of which this entity was discovered');
-    table.index('entity_ref', 'entity_ref_idx');
-    table.index('next_update_at', 'next_update_at_idx');
+    table.index('entity_ref', 'refresh_state_entity_ref_idx');
+    table.index('next_update_at', 'refresh_state_next_update_at_idx');
+  });
+
+  await knex.schema.createTable('refresh_state_references', table => {
+    table.comment(
+      'Holds edges between refresh state rows. Every time when an entity is processed and emits another entity, an edge will be stored to represent that fact. This is used to detect orphans and ultimately deletions.',
+    );
+    table
+      .text('source_special_key')
+      .nullable()
+      .comment(
+        'When the reference source is not an entity, this is an opaque identifier for that source.',
+      );
+    table
+      .text('source_entity_id')
+      .nullable()
+      .references('entity_id')
+      .inTable('refresh_state')
+      .onDelete('CASCADE')
+      .comment(
+        'When the reference source is an entity, this is the ID of the source entity.',
+      );
+    table
+      .text('target_entity_id')
+      .notNullable()
+      .references('entity_id')
+      .inTable('refresh_state')
+      .onDelete('CASCADE')
+      .comment('The ID of the target entity.');
+    table.index(
+      'source_special_key',
+      'refresh_state_references_source_special_key_idx',
+    );
+    table.index(
+      'source_entity_id',
+      'refresh_state_references_source_entity_id_idx',
+    );
+    table.index(
+      'target_entity_id',
+      'refresh_state_references_target_entity_id_idx',
+    );
   });
 };
 
@@ -67,9 +161,15 @@ exports.up = async function up(knex) {
  * @param {import('knex').Knex} knex
  */
 exports.down = async function down(knex) {
+  await knex.schema.alterTable('refresh_state_references', table => {
+    table.dropIndex([], 'refresh_state_references_source_special_key_idx');
+    table.dropIndex([], 'refresh_state_references_source_entity_id_idx');
+    table.dropIndex([], 'refresh_state_references_target_entity_id_idx');
+  });
   await knex.schema.alterTable('refresh_state', table => {
-    table.dropIndex([], 'entity_ref_idx');
-    table.dropIndex([], 'next_update_at_idx');
+    table.dropIndex([], 'refresh_state_entity_ref_idx');
+    table.dropIndex([], 'refresh_state_next_update_at_idx');
   });
   await knex.schema.dropTable('refresh_state');
+  await knex.schema.dropTable('references');
 };