paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added backend tests

Browse Source 
Signed-off-by: Andre Wanlin <andrewanlin@gmail.com>
This commit is contained in:
Andre Wanlin
2023-03-18 16:21:20 -05:00
parent 21403149d2
commit bbf91840a5
7 changed files with 562 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/shaggy-gorillas-occur.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-linguist-backend': patch
---
Added tests for the `LinguistBackendDatabase` and `LinguistBackendApi` which included refactoring to better support using SQLite and removed the default from the `processes_date` column
plugins/linguist-backend/api-report.md
+12
View File
@@ -3,6 +3,7 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { EntitiesOverview } from '@backstage/plugin-linguist-common';
import { EntityResults } from '@backstage/plugin-linguist-common';
import express from 'express';
import { HumanDuration } from '@backstage/types';
@@ -13,6 +14,7 @@ import { PluginDatabaseManager } from '@backstage/backend-common';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { ProcessedEntity } from '@backstage/plugin-linguist-common';
import { Results } from 'linguist-js/dist/types';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
import { TokenManager } from '@backstage/backend-common';
import { UrlReader } from '@backstage/backend-common';
@@ -38,8 +40,18 @@ export class LinguistBackendApi {
linguistJsOptions?: Record<string, unknown>,
);
// (undocumented)
addNewEntities(): Promise<void>;
// (undocumented)
generateEntitiesLanguages(): Promise<void>;
// (undocumented)
generateEntityLanguages(entityRef: string, url: string): Promise<string>;
// (undocumented)
getEntitiesOverview(): Promise<EntitiesOverview>;
// (undocumented)
getEntityLanguages(entityRef: string): Promise<Languages>;
// (undocumented)
getLinguistResults(dir: string): Promise<Results>;
// (undocumented)
processEntities(): Promise<void>;
}
plugins/linguist-backend/migrations/20230216_remove_processed_date_default.js
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* @param {import('knex').Knex} knex
*/
exports.up = async function up(knex) {
// Sqlite does not support this raw SQL
if (!knex.client.config.client.includes('sqlite3')) {
await knex.raw(
'ALTER TABLE entity_result ALTER COLUMN processed_date DROP DEFAULT;',
);
}
};
/**
* @param {import('knex').Knex} knex
*/
exports.down = async function down(knex) {
// Sqlite does not support this raw SQL
if (!knex.client.config.client.includes('sqlite3')) {
await knex.raw(
'ALTER TABLE entity_result ALTER COLUMN processed_date SET DEFAULT now();',
);
}
};
plugins/linguist-backend/src/api/LinguistBackendApi.test.ts
+308 -1
View File
@@ -13,7 +13,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { kindOrDefault } from './LinguistBackendApi';
import {
getVoidLogger,
PluginEndpointDiscovery,
ReadTreeResponse,
ServerTokenManager,
UrlReader,
} from '@backstage/backend-common';
import { GetEntitiesResponse } from '@backstage/catalog-client';
import { Results } from 'linguist-js/dist/types';
import { DateTime } from 'luxon';
import { LinguistBackendStore } from '../db';
import { kindOrDefault, LinguistBackendApi } from './LinguistBackendApi';
import fs from 'fs-extra';
const linguistResultMock = Promise.resolve({
files: {
count: 4,
bytes: 6010,
results: {
'/src/index.ts': 'TypeScript',
'/src/cli.js': 'JavaScript',
'/readme.md': 'Markdown',
'/no-lang': null,
},
},
languages: {
count: 3,
bytes: 6000,
results: {
JavaScript: { type: 'programming', bytes: 1000, color: '#f1e05a' },
TypeScript: { type: 'programming', bytes: 2000, color: '#2b7489' },
Markdown: { type: 'prose', bytes: 3000, color: '#083fa1' },
},
},
unknown: {
count: 1,
bytes: 10,
filenames: {
'no-lang': 10,
},
extensions: {},
},
} as Results);
describe('kindOrDefault', () => {
it('should return default kind when undefined', () => {
@@ -26,3 +68,268 @@ describe('kindOrDefault', () => {
expect(kindOrDefault(['API'])).toEqual(['API']);
});
});
describe('Linguist backend API', () => {
const getEntitiesMock = jest.fn();
jest.mock('@backstage/catalog-client', () => {
return {
CatalogClient: jest
.fn()
.mockImplementation(() => ({ getEntities: getEntitiesMock })),
};
});
const logger = getVoidLogger();
const store: jest.Mocked<LinguistBackendStore> = {
insertEntityResults: jest.fn(),
insertNewEntity: jest.fn(),
getEntityResults: jest.fn(),
getProcessedEntities: jest.fn(),
getUnprocessedEntities: jest.fn(),
};
const urlReader: jest.Mocked<UrlReader> = {
readTree: jest.fn(),
search: jest.fn(),
readUrl: jest.fn(),
};
const discovery: jest.Mocked<PluginEndpointDiscovery> = {
getBaseUrl: jest.fn(),
getExternalBaseUrl: jest.fn(),
};
const tokenManager = ServerTokenManager.noop();
const api = new LinguistBackendApi(
logger,
store,
urlReader,
discovery,
tokenManager,
);
beforeEach(() => {
jest.resetAllMocks();
});
it('should get languages for an entity', async () => {
store.getEntityResults.mockResolvedValue({
languageCount: 1,
totalBytes: 2205,
processedDate: '2023-02-15T20:10:21.378Z',
breakdown: [
{
name: 'YAML',
percentage: 100,
bytes: 2205,
type: 'data',
color: '#cb171e',
},
],
});
const entityRef = 'template:default/create-react-app-template';
const languages = await api.getEntityLanguages(entityRef);
expect(languages).toEqual({
languageCount: 1,
totalBytes: 2205,
processedDate: '2023-02-15T20:10:21.378Z',
breakdown: [
{
name: 'YAML',
percentage: 100,
bytes: 2205,
type: 'data',
color: '#cb171e',
},
],
});
});
it('should add new entities', async () => {
const testEntityListResponse: GetEntitiesResponse = {
items: [
{
apiVersion: 'backstage.io/v1beta1',
metadata: {
name: 'service-one',
},
kind: 'Component',
},
{
apiVersion: 'backstage.io/v1beta1',
metadata: {
name: 'service-two',
},
kind: 'Component',
},
{
apiVersion: 'backstage.io/v1beta1',
metadata: {
name: 'service-three',
},
kind: 'Component',
},
],
};
getEntitiesMock.mockResolvedValue(testEntityListResponse);
await api.addNewEntities();
expect(store.insertNewEntity).toHaveBeenCalledTimes(3);
});
it('should get default entity overview', async () => {
store.getProcessedEntities.mockResolvedValue([
{
entityRef: 'component:default/service-one',
processedDate: DateTime.now().toJSDate(),
},
{
entityRef: 'component:default/stale-service-two',
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
},
]);
store.getUnprocessedEntities.mockResolvedValue([
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
const overview = await api.getEntitiesOverview();
expect(overview.entityCount).toEqual(5);
expect(overview.processedCount).toEqual(2);
expect(overview.staleCount).toEqual(0);
expect(overview.pendingCount).toEqual(3);
expect(overview.filteredEntities).toEqual([
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
});
it('should get entity overview with stale items', async () => {
const staleApi = new LinguistBackendApi(
logger,
store,
urlReader,
discovery,
tokenManager,
{ days: 5 },
);
store.getProcessedEntities.mockResolvedValue([
{
entityRef: 'component:default/service-one',
processedDate: DateTime.now().toJSDate(),
},
{
entityRef: 'component:default/stale-service-two',
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
},
]);
store.getUnprocessedEntities.mockResolvedValue([
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
const overview = await staleApi.getEntitiesOverview();
expect(overview.entityCount).toEqual(5);
expect(overview.processedCount).toEqual(2);
expect(overview.staleCount).toEqual(1);
expect(overview.pendingCount).toEqual(4);
expect(overview.filteredEntities).toEqual([
'component:default/stale-service-two',
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
});
it('should generate and save languages for an entity', async () => {
const spy = jest
.spyOn(api, 'getLinguistResults')
.mockImplementation(() => linguistResultMock);
urlReader.readTree.mockResolvedValueOnce({
files: async () => [
{
content: async () => Buffer.from('-- XXX: code-data', 'utf8'),
path: 'my-file.js',
},
],
dir: async () => '/temp/my-code',
} as ReadTreeResponse);
const fsSpy = jest.spyOn(fs, 'remove');
await api.generateEntityLanguages(
'component:default/fake-service',
'https://some.fake/service/',
);
expect(api.getLinguistResults).toHaveBeenCalled();
expect(store.insertEntityResults).toHaveBeenCalled();
expect(fs.remove).toHaveBeenCalled();
spy.mockClear();
fsSpy.mockClear();
});
it('should generate languages for multiple entities using default', async () => {
store.getProcessedEntities.mockResolvedValue([
{
entityRef: 'component:default/service-one',
processedDate: DateTime.now().toJSDate(),
},
{
entityRef: 'component:default/stale-service-two',
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
},
]);
store.getUnprocessedEntities.mockResolvedValue([
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
const generateEntityLanguages = jest.spyOn(api, 'generateEntityLanguages');
await api.generateEntitiesLanguages();
expect(generateEntityLanguages).toHaveBeenCalledTimes(3);
});
it('should generate languages for multiple entities using defined batch size', async () => {
const batchApi = new LinguistBackendApi(
logger,
store,
urlReader,
discovery,
tokenManager,
undefined,
1,
);
store.getProcessedEntities.mockResolvedValue([
{
entityRef: 'component:default/service-one',
processedDate: DateTime.now().toJSDate(),
},
{
entityRef: 'component:default/stale-service-two',
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
},
]);
store.getUnprocessedEntities.mockResolvedValue([
'component:default/service-three',
'component:default/service-four',
'component:default/service-five',
]);
const generateEntityLanguages = jest.spyOn(
batchApi,
'generateEntityLanguages',
);
await batchApi.generateEntitiesLanguages();
expect(generateEntityLanguages).toHaveBeenCalledTimes(1);
});
});
plugins/linguist-backend/src/api/LinguistBackendApi.ts
+11 -6
View File
@@ -99,7 +99,7 @@ export class LinguistBackendApi {
await this.generateEntitiesLanguages();
}
private async addNewEntities() {
public async addNewEntities() {
const annotationKey = this.useSourceLocation
? ANNOTATION_SOURCE_LOCATION
: LINGUIST_ANNOTATION;
@@ -121,7 +121,7 @@ export class LinguistBackendApi {
});
}
private async generateEntitiesLanguages() {
public async generateEntitiesLanguages() {
const entitiesOverview = await this.getEntitiesOverview();
this.logger?.info(
`Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`,
@@ -156,7 +156,7 @@ export class LinguistBackendApi {
});
}
private async getEntitiesOverview(): Promise<EntitiesOverview> {
public async getEntitiesOverview(): Promise<EntitiesOverview> {
this.logger?.debug('Getting pending entities');
const processedEntities = await this.store.getProcessedEntities();
@@ -172,7 +172,7 @@ export class LinguistBackendApi {
const filteredEntities = staleEntities.concat(unprocessedEntities);
const entitiesOverview: EntitiesOverview = {
entityCount: unprocessedEntities.length,
entityCount: unprocessedEntities.length + processedEntities.length,
processedCount: processedEntities.length,
staleCount: staleEntities.length,
pendingCount: filteredEntities.length,
@@ -182,7 +182,7 @@ export class LinguistBackendApi {
return entitiesOverview;
}
private async generateEntityLanguages(
public async generateEntityLanguages(
entityRef: string,
url: string,
): Promise<string> {
@@ -193,7 +193,7 @@ export class LinguistBackendApi {
const readTreeResponse = await this.urlReader.readTree(url);
const dir = await readTreeResponse.dir();
const results = await linguist(dir, this.linguistJsOptions);
const results = await this.getLinguistResults(dir);
try {
const totalBytes = results.languages.bytes;
@@ -233,6 +233,11 @@ export class LinguistBackendApi {
await fs.remove(dir);
}
}
public async getLinguistResults(dir: string) {
const results = await linguist(dir, this.linguistJsOptions);
return results;
}
}
export function kindOrDefault(kind?: string[]) {
plugins/linguist-backend/src/db/LinguistBackendDatabase.test.ts
+169
View File
@@ -0,0 +1,169 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Knex as KnexType, Knex } from 'knex';
import { TestDatabases } from '@backstage/backend-test-utils';
import {
LinguistBackendDatabase,
LinguistBackendStore,
} from './LinguistBackendDatabase';
import { Languages, ProcessedEntity } from '@backstage/plugin-linguist-common';
function createDatabaseManager(
client: KnexType,
skipMigrations: boolean = false,
) {
return {
getClient: async () => client,
migrations: {
skip: skipMigrations,
},
};
}
const rawDbEntityResultRows = [
{
id: '14b32439-848a-49b2-92a4-98753f932606',
entity_ref: 'template:default/create-react-app-template',
languages: undefined,
processed_date: undefined,
},
{
id: '8c85d3ec-ccea-4b29-9de9-ccdd7e5ba452',
entity_ref: 'template:default/docs-template',
languages: undefined,
processed_date: undefined,
},
{
id: 'b922c87b-37bc-4505-b4af-70a1438decda',
entity_ref: 'template:default/pull-request',
languages:
'{"languageCount":1,"totalBytes":2205,"processedDate":"2023-02-15T20:10:21.378Z","breakdown":[{"name":"YAML","percentage":100,"bytes":2205,"type":"data","color":"#cb171e"}]}',
processed_date: new Date('2023-02-15 20:10:21.378Z'),
},
{
id: 'bd555e6d-a3d0-4b48-a930-194db8f80db7',
entity_ref: 'template:default/react-ssr-template',
languages:
'{"languageCount":8,"totalBytes":8988,"processedDate":"2023-02-15T20:10:21.388Z","breakdown":[{"name":"INI","percentage":2.26,"bytes":203,"type":"data","color":"#d1dbe0"},{"name":"JavaScript","percentage":5.94,"bytes":534,"type":"programming","color":"#f1e05a"},{"name":"YAML","percentage":31.09,"bytes":2794,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":11.79,"bytes":1060,"type":"prose","color":"#083fa1"},{"name":"JSON","percentage":21.09,"bytes":1896,"type":"data","color":"#292929"},{"name":"CSS","percentage":0,"bytes":0,"type":"markup","color":"#563d7c"},{"name":"TSX","percentage":26.01,"bytes":2338,"type":"programming","color":"#3178c6"},{"name":"TypeScript","percentage":1.81,"bytes":163,"type":"programming","color":"#3178c6"}]}',
processed_date: new Date('2023-02-15 20:10:21.388Z'),
},
{
id: '4145c0cf-44e9-4e95-9d57-781af4685b28',
entity_ref: 'template:default/springboot-template',
languages:
'{"languageCount":9,"totalBytes":80986,"processedDate":"2023-02-15T20:10:21.419Z","breakdown":[{"name":"Shell","percentage":0.16,"bytes":130,"type":"programming","color":"#89e051"},{"name":"INI","percentage":0.35,"bytes":286,"type":"data","color":"#d1dbe0"},{"name":"Dockerfile","percentage":0.3,"bytes":246,"type":"programming","color":"#384d54"},{"name":"YAML","percentage":3.92,"bytes":3171,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":1.31,"bytes":1059,"type":"prose","color":"#083fa1"},{"name":"XML","percentage":10.48,"bytes":8491,"type":"data","color":"#0060ac"},{"name":"Java","percentage":1.8,"bytes":1455,"type":"programming","color":"#b07219"},{"name":"Text","percentage":81.22,"bytes":65780,"type":"prose"},{"name":"Protocol Buffer","percentage":0.45,"bytes":368,"type":"data"}]}',
processed_date: new Date('2023-02-15 20:10:21.419Z'),
},
];
describe('Linguist database', () => {
const databases = TestDatabases.create();
let store: LinguistBackendStore;
let testDbClient: Knex<any, unknown[]>;
beforeAll(async () => {
testDbClient = await databases.init('SQLITE_3');
const database = createDatabaseManager(testDbClient);
store = await LinguistBackendDatabase.create(await database.getClient());
});
beforeEach(async () => {
await testDbClient.batchInsert('entity_result', rawDbEntityResultRows);
});
afterEach(async () => {
await testDbClient('entity_result').delete();
});
it('should be able to return entity results', async () => {
const validLanguagesResult: Languages = {
languageCount: 1,
totalBytes: 2205,
processedDate: '2023-02-15T20:10:21.378Z',
breakdown: [
{
name: 'YAML',
percentage: 100,
bytes: 2205,
type: 'data',
color: '#cb171e',
},
],
};
const entityResult = await store.getEntityResults(
'template:default/pull-request',
);
expect(entityResult).toMatchObject(validLanguagesResult);
});
it('should return empty entity results when not found', async () => {
const validEmptyLanguagesResult: Languages = {
languageCount: 0,
totalBytes: 0,
processedDate: 'undefined',
breakdown: [],
};
const entityResult = await store.getEntityResults(
'template:default/create-react-app-template',
);
expect(entityResult).toMatchObject(validEmptyLanguagesResult);
});
it('should be able to return unprocessed entities', async () => {
const validUnprocessedEntities: string[] = [
'template:default/create-react-app-template',
'template:default/docs-template',
];
const unprocessedEntities = await store.getUnprocessedEntities();
expect(unprocessedEntities).toMatchObject(validUnprocessedEntities);
});
it('should be able to return processed entities', async () => {
const validProcessedEntities: ProcessedEntity[] = [
{
entityRef: 'template:default/pull-request',
processedDate: new Date('2023-02-15 20:10:21.378Z'),
},
{
entityRef: 'template:default/react-ssr-template',
processedDate: new Date('2023-02-15 20:10:21.388Z'),
},
{
entityRef: 'template:default/springboot-template',
processedDate: new Date('2023-02-15 20:10:21.419Z'),
},
];
const processedEntities = await store.getProcessedEntities();
expect(processedEntities).toMatchObject(validProcessedEntities);
});
it('should insert new entities and ignore duplicates', async () => {
const before = testDbClient.count('entity_result');
await store.insertNewEntity('component:/default/new-entity-one');
await store.insertNewEntity('component:/default/new-entity-two');
await store.insertNewEntity('template:default/pull-request');
const after = testDbClient.count('entity_result');
expect(before).toEqual(after);
});
});
plugins/linguist-backend/src/db/LinguistBackendDatabase.ts
+18 -4
View File
@@ -26,8 +26,8 @@ import {
export type RawDbEntityResultRow = {
id: string;
entity_ref: string;
languages: string;
processed_date: Date;
languages?: string;
processed_date?: Date;
};
/** @public */
@@ -102,7 +102,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
}
try {
return JSON.parse(entityResults.languages);
return JSON.parse(entityResults.languages as string);
} catch (error) {
throw new Error(`Failed to parse languages for '${entityRef}', ${error}`);
}
@@ -118,9 +118,20 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
}
const processedEntities = rawEntities.map(rawEntity => {
// Note: processed_date should never be null, this is handled by the DB query above
let processedDate = new Date();
if (rawEntity.processed_date) {
// SQLite will return a Timestamp whereas Postgres will return a proper Date
// This tests to see if we are getting a timestamp and convert if needed
processedDate = new Date(+rawEntity.processed_date.toString());
if (isNaN(+rawEntity.processed_date.toString())) {
processedDate = rawEntity.processed_date;
}
}
const processEntity = {
entityRef: rawEntity.entity_ref,
processedDate: rawEntity.processed_date,
processedDate: processedDate,
};
return processEntity;
@@ -131,6 +142,9 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
async getUnprocessedEntities(): Promise<string[] | []> {
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result')
// TODO(ahhhndre) processed_date should always be null as well but it had a default to the current date
// once the default has been removed and released, we can then come back an enable this check
// .whereNull('processed_date')
.whereNull('languages')
.orderBy('created_at', 'asc');