diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 8b17b997dd..1b2c4af832 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -58,6 +58,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // (undocumented) static create(knex: Knex): Promise; // (undocumented) + deleteEntity(entityRef: string): Promise; + // (undocumented) + getAllEntities(): Promise; + // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) getProcessedEntities(): Promise; @@ -71,6 +75,10 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // @public (undocumented) export interface LinguistBackendStore { + // (undocumented) + deleteEntity(entityRef: string): Promise; + // (undocumented) + getAllEntities(): Promise; // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts index 35336bb5e9..1b0a4f4673 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.test.ts @@ -79,6 +79,8 @@ describe('Linguist backend API', () => { getEntityResults: jest.fn(), getProcessedEntities: jest.fn(), getUnprocessedEntities: jest.fn(), + getAllEntities: jest.fn(), + deleteEntity: jest.fn(), }; const urlReader: jest.Mocked = { @@ -140,7 +142,7 @@ describe('Linguist backend API', () => { }); }); - it('should add new entities', async () => { + it('should insert new entities', async () => { const testEntityListResponse: GetEntitiesResponse = { items: [ { @@ -172,6 +174,24 @@ describe('Linguist backend API', () => { expect(store.insertNewEntity).toHaveBeenCalledTimes(3); }); + it('should delete entities not in Catalog', async () => { + store.getAllEntities.mockResolvedValue([ + 'component:default/service-one', + 'component:default/stale-service-two', + ]); + + catalogApi.getEntityByRef.mockResolvedValueOnce({ + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + }, + kind: 'Component', + }); + + await api.cleanEntities(); + expect(store.deleteEntity).toHaveBeenCalledTimes(1); + }); + it('should get default entity overview', async () => { store.getProcessedEntities.mockResolvedValue([ { diff --git a/plugins/linguist-backend/src/api/LinguistBackendClient.ts b/plugins/linguist-backend/src/api/LinguistBackendClient.ts index a2f6e1ef67..0bd6370ac3 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendClient.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendClient.ts @@ -92,11 +92,12 @@ export class LinguistBackendClient implements LinguistBackendApi { async processEntities(): Promise { this.logger?.info('Updating list of entities'); - await this.addNewEntities(); - this.logger?.info('Processing applicable entities through Linguist'); + this.logger?.info('Cleaning list of entities'); + await this.cleanEntities(); + this.logger?.info('Processing applicable entities through Linguist'); await this.generateEntitiesLanguages(); } @@ -123,6 +124,23 @@ export class LinguistBackendClient implements LinguistBackendApi { }); } + /** @internal */ + async cleanEntities(): Promise { + this.logger?.info('Cleaning entities in Linguist queue'); + const allEntities = await this.store.getAllEntities(); + + for (const entityRef of allEntities) { + const result = await this.catalogApi.getEntityByRef(entityRef); + + if (!result) { + this.logger?.info( + `Entity ${entityRef} was not found in the Catalog, it will be deleted`, + ); + await this.store.deleteEntity(entityRef); + } + } + } + /** @internal */ async generateEntitiesLanguages(): Promise { const entitiesOverview = await this.getEntitiesOverview(); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts index 1ab092b953..5998ddfc1e 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts @@ -170,14 +170,31 @@ describe('Linguist database', () => { }); it('should insert new entities and ignore duplicates', async () => { - const before = testDbClient.count('entity_result'); + const before = testDbClient.from('entity_result').count(); await store.insertNewEntity('component:/default/new-entity-one'); await store.insertNewEntity('component:/default/new-entity-two'); await store.insertNewEntity('template:default/pull-request'); - const after = testDbClient.count('entity_result'); + const after = testDbClient.from('entity_result').count(); expect(before).toEqual(after); }); + + it('should get all entities', async () => { + const allEntities = await store.getAllEntities(); + + expect(allEntities.length).toEqual(5); + }); + + it('should delete entity by its entityRef', async () => { + const before = await testDbClient.from('entity_result').count(); + + await store.deleteEntity('template:default/create-react-app-template'); + + const after = await testDbClient.from('entity_result').count(); + + expect(before).toEqual([{ 'count(*)': 5 }]); + expect(after).toEqual([{ 'count(*)': 4 }]); + }); }); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index 5a165695da..2a15b2d823 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -37,6 +37,8 @@ export interface LinguistBackendStore { getEntityResults(entityRef: string): Promise; getProcessedEntities(): Promise; getUnprocessedEntities(): Promise; + getAllEntities(): Promise; + deleteEntity(entityRef: string): Promise; } const migrationsDir = resolvePackagePath( @@ -158,4 +160,24 @@ export class LinguistBackendDatabase implements LinguistBackendStore { return unprocessedEntities; } + + async getAllEntities(): Promise { + const rawEntities = await this.db('entity_result'); + + if (!rawEntities) { + return []; + } + + const allEntities = rawEntities.map(rawEntity => { + return rawEntity.entity_ref; + }); + + return allEntities; + } + + async deleteEntity(entityRef: string): Promise { + await this.db('entity_result') + .where('entity_ref', entityRef) + .delete(); + } }