Clean up orphan entities

Signed-off-by: Andre Wanlin <andrewanlin@gmail.com>

plugins/linguist-backend/api-report.md

@@ -58,6 +58,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
   // (undocumented)
   static create(knex: Knex): Promise<LinguistBackendStore>;
   // (undocumented)
+  deleteEntity(entityRef: string): Promise<void>;
+  // (undocumented)
+  getAllEntities(): Promise<string[]>;
+  // (undocumented)
   getEntityResults(entityRef: string): Promise<Languages>;
   // (undocumented)
   getProcessedEntities(): Promise<ProcessedEntity[]>;
@@ -71,6 +75,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
 
 // @public (undocumented)
 export interface LinguistBackendStore {
+  // (undocumented)
+  deleteEntity(entityRef: string): Promise<void>;
+  // (undocumented)
+  getAllEntities(): Promise<string[]>;
   // (undocumented)
   getEntityResults(entityRef: string): Promise<Languages>;
   // (undocumented)

plugins/linguist-backend/src/api/LinguistBackendClient.test.ts

@@ -79,6 +79,8 @@ describe('Linguist backend API', () => {
     getEntityResults: jest.fn(),
     getProcessedEntities: jest.fn(),
     getUnprocessedEntities: jest.fn(),
+    getAllEntities: jest.fn(),
+    deleteEntity: jest.fn(),
   };
 
   const urlReader: jest.Mocked<UrlReader> = {
@@ -140,7 +142,7 @@ describe('Linguist backend API', () => {
     });
   });
 
-  it('should add new entities', async () => {
+  it('should insert new entities', async () => {
     const testEntityListResponse: GetEntitiesResponse = {
       items: [
         {
@@ -172,6 +174,24 @@ describe('Linguist backend API', () => {
     expect(store.insertNewEntity).toHaveBeenCalledTimes(3);
   });
 
+  it('should delete entities not in Catalog', async () => {
+    store.getAllEntities.mockResolvedValue([
+      'component:default/service-one',
+      'component:default/stale-service-two',
+    ]);
+
+    catalogApi.getEntityByRef.mockResolvedValueOnce({
+      apiVersion: 'backstage.io/v1beta1',
+      metadata: {
+        name: 'service-one',
+      },
+      kind: 'Component',
+    });
+
+    await api.cleanEntities();
+    expect(store.deleteEntity).toHaveBeenCalledTimes(1);
+  });
+
   it('should get default entity overview', async () => {
     store.getProcessedEntities.mockResolvedValue([
       {

plugins/linguist-backend/src/api/LinguistBackendClient.ts

@@ -92,11 +92,12 @@ export class LinguistBackendClient implements LinguistBackendApi {
 
   async processEntities(): Promise<void> {
     this.logger?.info('Updating list of entities');
-
     await this.addNewEntities();
 
-    this.logger?.info('Processing applicable entities through Linguist');
+    this.logger?.info('Cleaning list of entities');
+    await this.cleanEntities();
 
+    this.logger?.info('Processing applicable entities through Linguist');
     await this.generateEntitiesLanguages();
   }
 
@@ -123,6 +124,23 @@ export class LinguistBackendClient implements LinguistBackendApi {
     });
   }
 
+  /** @internal */
+  async cleanEntities(): Promise<void> {
+    this.logger?.info('Cleaning entities in Linguist queue');
+    const allEntities = await this.store.getAllEntities();
+
+    for (const entityRef of allEntities) {
+      const result = await this.catalogApi.getEntityByRef(entityRef);
+
+      if (!result) {
+        this.logger?.info(
+          `Entity ${entityRef} was not found in the Catalog, it will be deleted`,
+        );
+        await this.store.deleteEntity(entityRef);
+      }
+    }
+  }
+
   /** @internal */
   async generateEntitiesLanguages(): Promise<void> {
     const entitiesOverview = await this.getEntitiesOverview();

plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts

@@ -170,14 +170,31 @@ describe('Linguist database', () => {
   });
 
   it('should insert new entities and ignore duplicates', async () => {
-    const before = testDbClient.count('entity_result');
+    const before = testDbClient.from('entity_result').count();
 
     await store.insertNewEntity('component:/default/new-entity-one');
     await store.insertNewEntity('component:/default/new-entity-two');
     await store.insertNewEntity('template:default/pull-request');
 
-    const after = testDbClient.count('entity_result');
+    const after = testDbClient.from('entity_result').count();
 
     expect(before).toEqual(after);
   });
+
+  it('should get all entities', async () => {
+    const allEntities = await store.getAllEntities();
+
+    expect(allEntities.length).toEqual(5);
+  });
+
+  it('should delete entity by its entityRef', async () => {
+    const before = await testDbClient.from('entity_result').count();
+
+    await store.deleteEntity('template:default/create-react-app-template');
+
+    const after = await testDbClient.from('entity_result').count();
+
+    expect(before).toEqual([{ 'count(*)': 5 }]);
+    expect(after).toEqual([{ 'count(*)': 4 }]);
+  });
 });

plugins/linguist-backend/src/db/LinguistBackendDatabase.ts

@@ -37,6 +37,8 @@ export interface LinguistBackendStore {
   getEntityResults(entityRef: string): Promise<Languages>;
   getProcessedEntities(): Promise<ProcessedEntity[]>;
   getUnprocessedEntities(): Promise<string[]>;
+  getAllEntities(): Promise<string[]>;
+  deleteEntity(entityRef: string): Promise<void>;
 }
 
 const migrationsDir = resolvePackagePath(
@@ -158,4 +160,24 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
 
     return unprocessedEntities;
   }
+
+  async getAllEntities(): Promise<string[]> {
+    const rawEntities = await this.db<RawDbEntityResultRow>('entity_result');
+
+    if (!rawEntities) {
+      return [];
+    }
+
+    const allEntities = rawEntities.map(rawEntity => {
+      return rawEntity.entity_ref;
+    });
+
+    return allEntities;
+  }
+
+  async deleteEntity(entityRef: string): Promise<void> {
+    await this.db<RawDbEntityResultRow>('entity_result')
+      .where('entity_ref', entityRef)
+      .delete();
+  }
 }