From bbf91840a52a1521cc7123187ac4b0ee7519afd9 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Sat, 18 Mar 2023 16:21:20 -0500 Subject: [PATCH] Added backend tests Signed-off-by: Andre Wanlin --- .changeset/shaggy-gorillas-occur.md | 5 + plugins/linguist-backend/api-report.md | 12 + .../20230216_remove_processed_date_default.js | 39 +++ .../src/api/LinguistBackendApi.test.ts | 309 +++++++++++++++++- .../src/api/LinguistBackendApi.ts | 17 +- .../src/db/LinguistBackendDatabase.test.ts | 169 ++++++++++ .../src/db/LinguistBackendDatabase.ts | 22 +- 7 files changed, 562 insertions(+), 11 deletions(-) create mode 100644 .changeset/shaggy-gorillas-occur.md create mode 100644 plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js create mode 100644 plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts diff --git a/.changeset/shaggy-gorillas-occur.md b/.changeset/shaggy-gorillas-occur.md new file mode 100644 index 0000000000..c122032e7f --- /dev/null +++ b/.changeset/shaggy-gorillas-occur.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-linguist-backend': patch +--- + +Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` which included refactoring to better support using SQLite and removed the default from the `processes_date` column diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index a47f123ac4..ca36a1f816 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,6 +3,7 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { EntitiesOverview } from '@backstage/plugin-linguist-common'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; import { HumanDuration } from '@backstage/types'; @@ -13,6 +14,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PluginTaskScheduler } from '@backstage/backend-tasks'; import { ProcessedEntity } from '@backstage/plugin-linguist-common'; +import { Results } from 'linguist-js/dist/types'; import { TaskScheduleDefinition } from '@backstage/backend-tasks'; import { TokenManager } from '@backstage/backend-common'; import { UrlReader } from '@backstage/backend-common'; @@ -38,8 +40,18 @@ export class LinguistBackendApi { linguistJsOptions?: Record, ); // (undocumented) + addNewEntities(): Promise; + // (undocumented) + generateEntitiesLanguages(): Promise; + // (undocumented) + generateEntityLanguages(entityRef: string, url: string): Promise; + // (undocumented) + getEntitiesOverview(): Promise; + // (undocumented) getEntityLanguages(entityRef: string): Promise; // (undocumented) + getLinguistResults(dir: string): Promise; + // (undocumented) processEntities(): Promise; } diff --git a/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js b/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js new file mode 100644 index 0000000000..5aaf02216e --- /dev/null +++ b/plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js @@ -0,0 +1,39 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + // Sqlite does not support this raw SQL + if (!knex.client.config.client.includes('sqlite3')) { + await knex.raw( + 'ALTER TABLE entity_result ALTER COLUMN processed_date DROP DEFAULT;', + ); + } +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + // Sqlite does not support this raw SQL + if (!knex.client.config.client.includes('sqlite3')) { + await knex.raw( + 'ALTER TABLE entity_result ALTER COLUMN processed_date SET DEFAULT now();', + ); + } +}; diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts index 741d52e734..baf7d27ad7 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.test.ts @@ -13,7 +13,49 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { kindOrDefault } from './LinguistBackendApi'; +import { + getVoidLogger, + PluginEndpointDiscovery, + ReadTreeResponse, + ServerTokenManager, + UrlReader, +} from '@backstage/backend-common'; +import { GetEntitiesResponse } from '@backstage/catalog-client'; +import { Results } from 'linguist-js/dist/types'; +import { DateTime } from 'luxon'; +import { LinguistBackendStore } from '../db'; +import { kindOrDefault, LinguistBackendApi } from './LinguistBackendApi'; +import fs from 'fs-extra'; + +const linguistResultMock = Promise.resolve({ + files: { + count: 4, + bytes: 6010, + results: { + '/src/index.ts': 'TypeScript', + '/src/cli.js': 'JavaScript', + '/readme.md': 'Markdown', + '/no-lang': null, + }, + }, + languages: { + count: 3, + bytes: 6000, + results: { + JavaScript: { type: 'programming', bytes: 1000, color: '#f1e05a' }, + TypeScript: { type: 'programming', bytes: 2000, color: '#2b7489' }, + Markdown: { type: 'prose', bytes: 3000, color: '#083fa1' }, + }, + }, + unknown: { + count: 1, + bytes: 10, + filenames: { + 'no-lang': 10, + }, + extensions: {}, + }, +} as Results); describe('kindOrDefault', () => { it('should return default kind when undefined', () => { @@ -26,3 +68,268 @@ describe('kindOrDefault', () => { expect(kindOrDefault(['API'])).toEqual(['API']); }); }); + +describe('Linguist backend API', () => { + const getEntitiesMock = jest.fn(); + jest.mock('@backstage/catalog-client', () => { + return { + CatalogClient: jest + .fn() + .mockImplementation(() => ({ getEntities: getEntitiesMock })), + }; + }); + + const logger = getVoidLogger(); + + const store: jest.Mocked = { + insertEntityResults: jest.fn(), + insertNewEntity: jest.fn(), + getEntityResults: jest.fn(), + getProcessedEntities: jest.fn(), + getUnprocessedEntities: jest.fn(), + }; + + const urlReader: jest.Mocked = { + readTree: jest.fn(), + search: jest.fn(), + readUrl: jest.fn(), + }; + + const discovery: jest.Mocked = { + getBaseUrl: jest.fn(), + getExternalBaseUrl: jest.fn(), + }; + + const tokenManager = ServerTokenManager.noop(); + + const api = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + ); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should get languages for an entity', async () => { + store.getEntityResults.mockResolvedValue({ + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }); + + const entityRef = 'template:default/create-react-app-template'; + const languages = await api.getEntityLanguages(entityRef); + expect(languages).toEqual({ + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }); + }); + + it('should add new entities', async () => { + const testEntityListResponse: GetEntitiesResponse = { + items: [ + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-one', + }, + kind: 'Component', + }, + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-two', + }, + kind: 'Component', + }, + { + apiVersion: 'backstage.io/v1beta1', + metadata: { + name: 'service-three', + }, + kind: 'Component', + }, + ], + }; + getEntitiesMock.mockResolvedValue(testEntityListResponse); + + await api.addNewEntities(); + expect(store.insertNewEntity).toHaveBeenCalledTimes(3); + }); + + it('should get default entity overview', async () => { + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + + const overview = await api.getEntitiesOverview(); + expect(overview.entityCount).toEqual(5); + expect(overview.processedCount).toEqual(2); + expect(overview.staleCount).toEqual(0); + expect(overview.pendingCount).toEqual(3); + expect(overview.filteredEntities).toEqual([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + }); + + it('should get entity overview with stale items', async () => { + const staleApi = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + { days: 5 }, + ); + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + + const overview = await staleApi.getEntitiesOverview(); + expect(overview.entityCount).toEqual(5); + expect(overview.processedCount).toEqual(2); + expect(overview.staleCount).toEqual(1); + expect(overview.pendingCount).toEqual(4); + expect(overview.filteredEntities).toEqual([ + 'component:default/stale-service-two', + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + }); + + it('should generate and save languages for an entity', async () => { + const spy = jest + .spyOn(api, 'getLinguistResults') + .mockImplementation(() => linguistResultMock); + + urlReader.readTree.mockResolvedValueOnce({ + files: async () => [ + { + content: async () => Buffer.from('-- XXX: code-data', 'utf8'), + path: 'my-file.js', + }, + ], + dir: async () => '/temp/my-code', + } as ReadTreeResponse); + + const fsSpy = jest.spyOn(fs, 'remove'); + + await api.generateEntityLanguages( + 'component:default/fake-service', + 'https://some.fake/service/', + ); + expect(api.getLinguistResults).toHaveBeenCalled(); + expect(store.insertEntityResults).toHaveBeenCalled(); + expect(fs.remove).toHaveBeenCalled(); + spy.mockClear(); + fsSpy.mockClear(); + }); + + it('should generate languages for multiple entities using default', async () => { + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + const generateEntityLanguages = jest.spyOn(api, 'generateEntityLanguages'); + await api.generateEntitiesLanguages(); + expect(generateEntityLanguages).toHaveBeenCalledTimes(3); + }); + + it('should generate languages for multiple entities using defined batch size', async () => { + const batchApi = new LinguistBackendApi( + logger, + store, + urlReader, + discovery, + tokenManager, + undefined, + 1, + ); + store.getProcessedEntities.mockResolvedValue([ + { + entityRef: 'component:default/service-one', + processedDate: DateTime.now().toJSDate(), + }, + { + entityRef: 'component:default/stale-service-two', + processedDate: DateTime.now().minus({ days: 45 }).toJSDate(), + }, + ]); + + store.getUnprocessedEntities.mockResolvedValue([ + 'component:default/service-three', + 'component:default/service-four', + 'component:default/service-five', + ]); + const generateEntityLanguages = jest.spyOn( + batchApi, + 'generateEntityLanguages', + ); + await batchApi.generateEntitiesLanguages(); + expect(generateEntityLanguages).toHaveBeenCalledTimes(1); + }); +}); diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.ts index df7e30a7b0..766012018f 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.ts @@ -99,7 +99,7 @@ export class LinguistBackendApi { await this.generateEntitiesLanguages(); } - private async addNewEntities() { + public async addNewEntities() { const annotationKey = this.useSourceLocation ? ANNOTATION_SOURCE_LOCATION : LINGUIST_ANNOTATION; @@ -121,7 +121,7 @@ export class LinguistBackendApi { }); } - private async generateEntitiesLanguages() { + public async generateEntitiesLanguages() { const entitiesOverview = await this.getEntitiesOverview(); this.logger?.info( `Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`, @@ -156,7 +156,7 @@ export class LinguistBackendApi { }); } - private async getEntitiesOverview(): Promise { + public async getEntitiesOverview(): Promise { this.logger?.debug('Getting pending entities'); const processedEntities = await this.store.getProcessedEntities(); @@ -172,7 +172,7 @@ export class LinguistBackendApi { const filteredEntities = staleEntities.concat(unprocessedEntities); const entitiesOverview: EntitiesOverview = { - entityCount: unprocessedEntities.length, + entityCount: unprocessedEntities.length + processedEntities.length, processedCount: processedEntities.length, staleCount: staleEntities.length, pendingCount: filteredEntities.length, @@ -182,7 +182,7 @@ export class LinguistBackendApi { return entitiesOverview; } - private async generateEntityLanguages( + public async generateEntityLanguages( entityRef: string, url: string, ): Promise { @@ -193,7 +193,7 @@ export class LinguistBackendApi { const readTreeResponse = await this.urlReader.readTree(url); const dir = await readTreeResponse.dir(); - const results = await linguist(dir, this.linguistJsOptions); + const results = await this.getLinguistResults(dir); try { const totalBytes = results.languages.bytes; @@ -233,6 +233,11 @@ export class LinguistBackendApi { await fs.remove(dir); } } + + public async getLinguistResults(dir: string) { + const results = await linguist(dir, this.linguistJsOptions); + return results; + } } export function kindOrDefault(kind?: string[]) { diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts new file mode 100644 index 0000000000..f4e0bba627 --- /dev/null +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts @@ -0,0 +1,169 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Knex as KnexType, Knex } from 'knex'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { + LinguistBackendDatabase, + LinguistBackendStore, +} from './LinguistBackendDatabase'; +import { Languages, ProcessedEntity } from '@backstage/plugin-linguist-common'; + +function createDatabaseManager( + client: KnexType, + skipMigrations: boolean = false, +) { + return { + getClient: async () => client, + migrations: { + skip: skipMigrations, + }, + }; +} + +const rawDbEntityResultRows = [ + { + id: '14b32439-848a-49b2-92a4-98753f932606', + entity_ref: 'template:default/create-react-app-template', + languages: undefined, + processed_date: undefined, + }, + { + id: '8c85d3ec-ccea-4b29-9de9-ccdd7e5ba452', + entity_ref: 'template:default/docs-template', + languages: undefined, + processed_date: undefined, + }, + { + id: 'b922c87b-37bc-4505-b4af-70a1438decda', + entity_ref: 'template:default/pull-request', + languages: + '{"languageCount":1,"totalBytes":2205,"processedDate":"2023-02-15T20:10:21.378Z","breakdown":[{"name":"YAML","percentage":100,"bytes":2205,"type":"data","color":"#cb171e"}]}', + processed_date: new Date('2023-02-15 20:10:21.378Z'), + }, + { + id: 'bd555e6d-a3d0-4b48-a930-194db8f80db7', + entity_ref: 'template:default/react-ssr-template', + languages: + '{"languageCount":8,"totalBytes":8988,"processedDate":"2023-02-15T20:10:21.388Z","breakdown":[{"name":"INI","percentage":2.26,"bytes":203,"type":"data","color":"#d1dbe0"},{"name":"JavaScript","percentage":5.94,"bytes":534,"type":"programming","color":"#f1e05a"},{"name":"YAML","percentage":31.09,"bytes":2794,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":11.79,"bytes":1060,"type":"prose","color":"#083fa1"},{"name":"JSON","percentage":21.09,"bytes":1896,"type":"data","color":"#292929"},{"name":"CSS","percentage":0,"bytes":0,"type":"markup","color":"#563d7c"},{"name":"TSX","percentage":26.01,"bytes":2338,"type":"programming","color":"#3178c6"},{"name":"TypeScript","percentage":1.81,"bytes":163,"type":"programming","color":"#3178c6"}]}', + processed_date: new Date('2023-02-15 20:10:21.388Z'), + }, + { + id: '4145c0cf-44e9-4e95-9d57-781af4685b28', + entity_ref: 'template:default/springboot-template', + languages: + '{"languageCount":9,"totalBytes":80986,"processedDate":"2023-02-15T20:10:21.419Z","breakdown":[{"name":"Shell","percentage":0.16,"bytes":130,"type":"programming","color":"#89e051"},{"name":"INI","percentage":0.35,"bytes":286,"type":"data","color":"#d1dbe0"},{"name":"Dockerfile","percentage":0.3,"bytes":246,"type":"programming","color":"#384d54"},{"name":"YAML","percentage":3.92,"bytes":3171,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":1.31,"bytes":1059,"type":"prose","color":"#083fa1"},{"name":"XML","percentage":10.48,"bytes":8491,"type":"data","color":"#0060ac"},{"name":"Java","percentage":1.8,"bytes":1455,"type":"programming","color":"#b07219"},{"name":"Text","percentage":81.22,"bytes":65780,"type":"prose"},{"name":"Protocol Buffer","percentage":0.45,"bytes":368,"type":"data"}]}', + processed_date: new Date('2023-02-15 20:10:21.419Z'), + }, +]; + +describe('Linguist database', () => { + const databases = TestDatabases.create(); + let store: LinguistBackendStore; + let testDbClient: Knex; + beforeAll(async () => { + testDbClient = await databases.init('SQLITE_3'); + const database = createDatabaseManager(testDbClient); + + store = await LinguistBackendDatabase.create(await database.getClient()); + }); + beforeEach(async () => { + await testDbClient.batchInsert('entity_result', rawDbEntityResultRows); + }); + afterEach(async () => { + await testDbClient('entity_result').delete(); + }); + + it('should be able to return entity results', async () => { + const validLanguagesResult: Languages = { + languageCount: 1, + totalBytes: 2205, + processedDate: '2023-02-15T20:10:21.378Z', + breakdown: [ + { + name: 'YAML', + percentage: 100, + bytes: 2205, + type: 'data', + color: '#cb171e', + }, + ], + }; + + const entityResult = await store.getEntityResults( + 'template:default/pull-request', + ); + expect(entityResult).toMatchObject(validLanguagesResult); + }); + + it('should return empty entity results when not found', async () => { + const validEmptyLanguagesResult: Languages = { + languageCount: 0, + totalBytes: 0, + processedDate: 'undefined', + breakdown: [], + }; + + const entityResult = await store.getEntityResults( + 'template:default/create-react-app-template', + ); + expect(entityResult).toMatchObject(validEmptyLanguagesResult); + }); + + it('should be able to return unprocessed entities', async () => { + const validUnprocessedEntities: string[] = [ + 'template:default/create-react-app-template', + 'template:default/docs-template', + ]; + + const unprocessedEntities = await store.getUnprocessedEntities(); + + expect(unprocessedEntities).toMatchObject(validUnprocessedEntities); + }); + + it('should be able to return processed entities', async () => { + const validProcessedEntities: ProcessedEntity[] = [ + { + entityRef: 'template:default/pull-request', + processedDate: new Date('2023-02-15 20:10:21.378Z'), + }, + { + entityRef: 'template:default/react-ssr-template', + processedDate: new Date('2023-02-15 20:10:21.388Z'), + }, + { + entityRef: 'template:default/springboot-template', + processedDate: new Date('2023-02-15 20:10:21.419Z'), + }, + ]; + + const processedEntities = await store.getProcessedEntities(); + + expect(processedEntities).toMatchObject(validProcessedEntities); + }); + + it('should insert new entities and ignore duplicates', async () => { + const before = testDbClient.count('entity_result'); + + await store.insertNewEntity('component:/default/new-entity-one'); + await store.insertNewEntity('component:/default/new-entity-two'); + await store.insertNewEntity('template:default/pull-request'); + + const after = testDbClient.count('entity_result'); + + expect(before).toEqual(after); + }); +}); diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index cf93374800..c045d0a4a9 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -26,8 +26,8 @@ import { export type RawDbEntityResultRow = { id: string; entity_ref: string; - languages: string; - processed_date: Date; + languages?: string; + processed_date?: Date; }; /** @public */ @@ -102,7 +102,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } try { - return JSON.parse(entityResults.languages); + return JSON.parse(entityResults.languages as string); } catch (error) { throw new Error(`Failed to parse languages for '${entityRef}', ${error}`); } @@ -118,9 +118,20 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } const processedEntities = rawEntities.map(rawEntity => { + // Note: processed_date should never be null, this is handled by the DB query above + let processedDate = new Date(); + if (rawEntity.processed_date) { + // SQLite will return a Timestamp whereas Postgres will return a proper Date + // This tests to see if we are getting a timestamp and convert if needed + processedDate = new Date(+rawEntity.processed_date.toString()); + if (isNaN(+rawEntity.processed_date.toString())) { + processedDate = rawEntity.processed_date; + } + } + const processEntity = { entityRef: rawEntity.entity_ref, - processedDate: rawEntity.processed_date, + processedDate: processedDate, }; return processEntity; @@ -131,6 +142,9 @@ export class LinguistBackendDatabase implements LinguistBackendStore { async getUnprocessedEntities(): Promise { const rawEntities = await this.db('entity_result') + // TODO(ahhhndre) processed_date should always be null as well but it had a default to the current date + // once the default has been removed and released, we can then come back an enable this check + // .whereNull('processed_date') .whereNull('languages') .orderBy('created_at', 'asc');