From 54f08f8bd9671bb87bf5b0932856bc1fd6dc25c5 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Thu, 9 Feb 2023 14:58:01 -0600 Subject: [PATCH] Refactored processing Signed-off-by: Andre Wanlin --- packages/backend/src/plugins/linguist.ts | 10 +- plugins/linguist-backend/README.md | 33 +++-- plugins/linguist-backend/api-report.md | 24 ++- .../migrations/20221115_init.js | 6 +- .../src/api/LinguistBackendApi.ts | 140 +++++++++++------- .../src/db/LinguistBackendDatabase.ts | 56 ++++++- .../src/service/router.test.ts | 3 +- .../linguist-backend/src/service/router.ts | 12 +- .../src/service/standaloneServer.ts | 3 +- plugins/linguist-common/api-report.md | 6 +- plugins/linguist-common/package.json | 4 - plugins/linguist-common/src/types.ts | 5 +- yarn.lock | 2 - 13 files changed, 199 insertions(+), 105 deletions(-) diff --git a/packages/backend/src/plugins/linguist.ts b/packages/backend/src/plugins/linguist.ts index 948ca7fbb5..2c8713a7e3 100644 --- a/packages/backend/src/plugins/linguist.ts +++ b/packages/backend/src/plugins/linguist.ts @@ -28,5 +28,13 @@ export default async function createPlugin( initialDelay: { seconds: 15 }, }; - return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env }); + return createRouter( + { + schedule: schedule, + age: { days: 30 }, + batchSize: 2, + useSourceLocation: false, + }, + { ...env }, + ); } diff --git a/plugins/linguist-backend/README.md b/plugins/linguist-backend/README.md index c42112b694..cd63f5db03 100644 --- a/plugins/linguist-backend/README.md +++ b/plugins/linguist-backend/README.md @@ -31,8 +31,8 @@ Here's how to get the backend up and running: env: PluginEnvironment, ): Promise { const schedule: TaskScheduleDefinition = { - frequency: { hours: 24 }, - timeout: { minutes: 30 }, + frequency: { minutes: 2 }, + timeout: { minutes: 15 }, initialDelay: { seconds: 90 }, }; @@ -57,20 +57,18 @@ Here's how to get the backend up and running: 4. Now run `yarn start-backend` from the repo root 5. Finally open `http://localhost:7007/api/linguist/health` in a browser and it should return `{"status":"ok"}` -## Scheduling +## Batch Size -The Linguist backend can be configured to generate the language breakdown for all the entities with the linguist annotation. This is done by passing a `TaskScheduleDefinition` to the `createRouter` function like in Step 2 of the [Up and Running](#up-and-running) documentation. - -Here's another example of the `TaskScheduleDefinition` that will run the language breakdown processing every 12 hours: +The Linguist backend is setup to process entities by acting as a queue where it will pull down all the applicable entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest entities that have not been processed to determine their languages and process them. To control the batch size simply provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this: ```ts -const schedule: TaskScheduleDefinition = { - frequency: { hours: 12 }, - timeout: { minutes: 15 }, - initialDelay: { seconds: 10 }, -}; +return createRouter({ schedule: schedule, batchSize: 40 }, { ...env }); ``` +**Note:** The default batch size is 20 + +## Refresh + The default setup will only generate the language breakdown for entities with the linguist annotation that have not been generated yet. If you want this process to also refresh the data you can do so by adding the `age` (as a `HumanDuration`) in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`: ```ts @@ -79,6 +77,19 @@ return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env }); With the `age` setup like this if the language breakdown is older than 15 days it will get regenerated. It's recommended that if you choose to use this configuration to set it to a large value - 30, 90, or 180 - as this data generally does not change drastically. +## Use Source Location + +You may wish to use the `backstage.io/source-location` annotation over using the `backstage.io/linguist` as you may not be able to quickly add that annotation to your Entities. To do this you'll just need to set the `useSourceLocation` boolean to `true` in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`: + +```ts +return createRouter( + { schedule: schedule, useSourceLocation: true }, + { ...env }, +); +``` + +**Note:** This has the potential to cause a lot of processing, be very thoughtful about this before hand + ## Links - [Frontend part of the plugin](../linguist/README.md) diff --git a/plugins/linguist-backend/api-report.md b/plugins/linguist-backend/api-report.md index 3fd2823815..484998b099 100644 --- a/plugins/linguist-backend/api-report.md +++ b/plugins/linguist-backend/api-report.md @@ -3,7 +3,6 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { Config } from '@backstage/config'; import { EntitiesOverview } from '@backstage/plugin-linguist-common'; import { EntityResults } from '@backstage/plugin-linguist-common'; import express from 'express'; @@ -27,12 +26,13 @@ export function createRouter( // @public (undocumented) export class LinguistBackendApi { constructor( - config: Config, logger: Logger, store: LinguistBackendStore, urlReader: UrlReader, discovery: PluginEndpointDiscovery, - age?: HumanDuration | undefined, + age?: HumanDuration, + batchSize?: number, + useSourceLocation?: boolean, ); // (undocumented) getEntitiesOverview(): Promise; @@ -52,9 +52,13 @@ export class LinguistBackendDatabase implements LinguistBackendStore { // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) - getProcessedEntities(): Promise; + getProcessedEntities(): Promise; + // (undocumented) + getUnprocessedEntities(): Promise; // (undocumented) insertEntityResults(entityLanguages: EntityResults): Promise; + // (undocumented) + insertNewEntity(entityRef: string): Promise; } // @public (undocumented) @@ -62,9 +66,13 @@ export interface LinguistBackendStore { // (undocumented) getEntityResults(entityRef: string): Promise; // (undocumented) - getProcessedEntities(): Promise; + getProcessedEntities(): Promise; + // (undocumented) + getUnprocessedEntities(): Promise; // (undocumented) insertEntityResults(entityLanguages: EntityResults): Promise; + // (undocumented) + insertNewEntity(entityRef: string): Promise; } // @public (undocumented) @@ -72,13 +80,15 @@ export interface PluginOptions { // (undocumented) age?: HumanDuration; // (undocumented) + batchSize?: number; + // (undocumented) schedule?: TaskScheduleDefinition; + // (undocumented) + useSourceLocation?: boolean; } // @public (undocumented) export interface RouterOptions { - // (undocumented) - config: Config; // (undocumented) database: PluginDatabaseManager; // (undocumented) diff --git a/plugins/linguist-backend/migrations/20221115_init.js b/plugins/linguist-backend/migrations/20221115_init.js index 9eb0a03a90..3bcf99e08b 100644 --- a/plugins/linguist-backend/migrations/20221115_init.js +++ b/plugins/linguist-backend/migrations/20221115_init.js @@ -36,10 +36,7 @@ exports.up = async function up(knex) { .unique() .notNullable() .comment('The entity ref that this Linguist result applies to'); - table - .text('languages') - .notNullable() - .comment('The results json as a string'); + table.text('languages').comment('The results json as a string'); table .dateTime('created_at') .defaultTo(knex.fn.now()) @@ -48,7 +45,6 @@ exports.up = async function up(knex) { table .dateTime('processed_date') .defaultTo(knex.fn.now()) - .notNullable() .comment('The timestamp when this entity was processed'); table.index('index', 'entity_result_index_idx'); table.index('entity_ref', 'entity_result_entity_ref_idx'); diff --git a/plugins/linguist-backend/src/api/LinguistBackendApi.ts b/plugins/linguist-backend/src/api/LinguistBackendApi.ts index 816d731d7f..a5bbc5e639 100644 --- a/plugins/linguist-backend/src/api/LinguistBackendApi.ts +++ b/plugins/linguist-backend/src/api/LinguistBackendApi.ts @@ -27,27 +27,48 @@ import { } from '@backstage/catalog-client'; import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common'; -import { Config } from '@backstage/config'; import { DateTime } from 'luxon'; import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common'; import { LinguistBackendStore } from '../db'; import { Logger } from 'winston'; import fs from 'fs-extra'; import linguist from 'linguist-js'; -import { stringifyEntityRef } from '@backstage/catalog-model'; +import { + ANNOTATION_SOURCE_LOCATION, + stringifyEntityRef, +} from '@backstage/catalog-model'; import { assertError } from '@backstage/errors'; import { HumanDuration } from '@backstage/types'; /** @public */ export class LinguistBackendApi { + private readonly logger: Logger; + private readonly store: LinguistBackendStore; + private readonly urlReader: UrlReader; + private readonly discovery: PluginEndpointDiscovery; + private readonly catalogClient: CatalogClient; + private readonly age?: HumanDuration; + private readonly batchSize?: number; + private readonly useSourceLocation?: boolean; public constructor( - private readonly config: Config, - private readonly logger: Logger, - private readonly store: LinguistBackendStore, - private readonly urlReader: UrlReader, - private readonly discovery: PluginEndpointDiscovery, - private readonly age?: HumanDuration, - ) {} + logger: Logger, + store: LinguistBackendStore, + urlReader: UrlReader, + discovery: PluginEndpointDiscovery, + age?: HumanDuration, + batchSize?: number, + useSourceLocation?: boolean, + ) { + this.logger = logger; + this.store = store; + this.urlReader = urlReader; + this.discovery = discovery; + this.catalogClient = new CatalogClient({ discoveryApi: this.discovery }); + + this.batchSize = batchSize; + this.age = age; + this.useSourceLocation = useSourceLocation; + } public async getEntityLanguages(entityRef: string): Promise { this.logger?.debug(`Getting languages for entity "${entityRef}"`); @@ -105,65 +126,59 @@ export class LinguistBackendApi { } public async processEntities() { + this.logger?.info('Updating list of entities'); + + await this.addNewEntities(); + this.logger?.info('Processing applicable entities through Linguist'); - const eo = await this.getEntitiesOverview(); + const entitiesOverview = await this.getEntitiesOverview(); this.logger?.info( - `Entities overview: Entity: ${eo.entityCount}, Processed: ${eo.processedCount}, Pending: ${eo.pendingCount}, Stale ${eo.staleCount}`, + `Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`, ); - const entities = eo.filteredEntities; - for (const key in entities) { - if (Object.prototype.hasOwnProperty.call(entities, key)) { - const entityRef = stringifyEntityRef(entities[key]); + const entities = entitiesOverview.filteredEntities.slice( + this.batchSize ?? 20, + ); + entities.forEach(async entityRef => { + const entity = await this.catalogClient.getEntityByRef(entityRef); + const annotationKey = this.useSourceLocation + ? ANNOTATION_SOURCE_LOCATION + : LINGUIST_ANNOTATION; - const url = - entities[key].metadata.annotations?.[LINGUIST_ANNOTATION] ?? ''; - - try { - await this.processEntity(entityRef, url); - } catch (e) { - assertError(e); - this.logger.error( - `Unable to process "${entityRef}" using "${url}", ${e.message}`, - ); - } + let url = entity?.metadata.annotations?.[annotationKey] ?? ''; + if (url.startsWith('url:')) { + url = url.slice(4); } - } + + try { + await this.processEntity(entityRef, url); + } catch (error) { + assertError(error); + this.logger.error( + `Unable to process "${entityRef}" using "${url}", message: ${error.message}, stack: ${error.stack}`, + ); + } + }); } public async getEntitiesOverview(): Promise { this.logger?.debug('Getting pending entities'); - const age = this.config.getOptionalNumber('linguist.age') ?? 0; - const catalogApi = new CatalogClient({ discoveryApi: this.discovery }); - const request: GetEntitiesRequest = { - filter: { - kind: ['API', 'Component', 'Template'], - [`metadata.annotations.${LINGUIST_ANNOTATION}`]: CATALOG_FILTER_EXISTS, - }, - fields: ['kind', 'metadata'], - }; - - const response = await catalogApi.getEntities(request); - const entities = response.items; const processedEntities = await this.store.getProcessedEntities(); + const staleEntities = processedEntities + .filter(pe => { + if (this.age === undefined) return false; + const staleDate = DateTime.now().minus(this.age as HumanDuration); + return DateTime.fromJSDate(pe.processedDate) <= staleDate; + }) + .map(pe => pe.entityRef); - const staleEntities = processedEntities.filter(pe => { - if (age === undefined) return false; - const staleDate = DateTime.now().minus(this.age as HumanDuration); - return DateTime.fromJSDate(pe.processed_date) >= staleDate; - }); - - const filteredEntities = entities.filter(en => { - return processedEntities.every(pe => { - const entityRef = stringifyEntityRef(en); - return pe.entityRef !== entityRef; - }); - }); + const unprocessedEntities = await this.store.getUnprocessedEntities(); + const filteredEntities = staleEntities.concat(unprocessedEntities); const entitiesOverview: EntitiesOverview = { - entityCount: entities.length, + entityCount: unprocessedEntities.length, processedCount: processedEntities.length, staleCount: staleEntities.length, pendingCount: filteredEntities.length, @@ -172,4 +187,25 @@ export class LinguistBackendApi { return entitiesOverview; } + + private async addNewEntities() { + const annotationKey = this.useSourceLocation + ? ANNOTATION_SOURCE_LOCATION + : LINGUIST_ANNOTATION; + const request: GetEntitiesRequest = { + filter: { + kind: ['API', 'Component', 'Template'], + [`metadata.annotations.${annotationKey}`]: CATALOG_FILTER_EXISTS, + }, + fields: ['kind', 'metadata'], + }; + + const response = await this.catalogClient.getEntities(request); + const entities = response.items; + + entities.forEach(entity => { + const entityRef = stringifyEntityRef(entity); + this.store.insertNewEntity(entityRef); + }); + } } diff --git a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts index 8faa5c3ff0..cf93374800 100644 --- a/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts +++ b/plugins/linguist-backend/src/db/LinguistBackendDatabase.ts @@ -33,8 +33,10 @@ export type RawDbEntityResultRow = { /** @public */ export interface LinguistBackendStore { insertEntityResults(entityLanguages: EntityResults): Promise; + insertNewEntity(entityRef: string): Promise; getEntityResults(entityRef: string): Promise; - getProcessedEntities(): Promise; + getProcessedEntities(): Promise; + getUnprocessedEntities(): Promise; } const migrationsDir = resolvePackagePath( @@ -71,8 +73,21 @@ export class LinguistBackendDatabase implements LinguistBackendStore { return result.id; } + async insertNewEntity(entityRef: string): Promise { + const entityLanguageId = uuid(); + + await this.db('entity_result') + .insert({ + id: entityLanguageId, + entity_ref: entityRef, + }) + .onConflict('entity_ref') + .ignore(); // If the entity_ref is in the table already then we don't want to add it again + } + async getEntityResults(entityRef: string): Promise { const entityResults = await this.db('entity_result') + .whereNotNull('languages') .where({ entity_ref: entityRef }) .first(); @@ -93,11 +108,40 @@ export class LinguistBackendDatabase implements LinguistBackendStore { } } - async getProcessedEntities(): Promise { - const entityResults = await this.db( - 'entity_result', - ).select('entity_ref', 'processed_date'); + async getProcessedEntities(): Promise { + const rawEntities = await this.db('entity_result') + .whereNotNull('processed_date') + .whereNotNull('languages'); - return entityResults; + if (!rawEntities) { + return []; + } + + const processedEntities = rawEntities.map(rawEntity => { + const processEntity = { + entityRef: rawEntity.entity_ref, + processedDate: rawEntity.processed_date, + }; + + return processEntity; + }); + + return processedEntities; + } + + async getUnprocessedEntities(): Promise { + const rawEntities = await this.db('entity_result') + .whereNull('languages') + .orderBy('created_at', 'asc'); + + if (!rawEntities) { + return []; + } + + const unprocessedEntities = rawEntities.map(rawEntity => { + return rawEntity.entity_ref; + }); + + return unprocessedEntities; } } diff --git a/plugins/linguist-backend/src/service/router.test.ts b/plugins/linguist-backend/src/service/router.test.ts index 1de162c97a..34f16f4b21 100644 --- a/plugins/linguist-backend/src/service/router.test.ts +++ b/plugins/linguist-backend/src/service/router.test.ts @@ -65,10 +65,9 @@ describe('createRouter', () => { beforeAll(async () => { const router = await createRouter( - { schedule: schedule, age: { days: 30 } }, + { schedule: schedule, age: { days: 30 }, useSourceLocation: false }, { linguistBackendApi, - config: new ConfigReader({}), discovery: testDiscovery, database: createDatabase(), reader: mockUrlReader, diff --git a/plugins/linguist-backend/src/service/router.ts b/plugins/linguist-backend/src/service/router.ts index 8cf426f2b3..e3aeec604e 100644 --- a/plugins/linguist-backend/src/service/router.ts +++ b/plugins/linguist-backend/src/service/router.ts @@ -20,7 +20,6 @@ import { PluginEndpointDiscovery, UrlReader, } from '@backstage/backend-common'; -import { Config } from '@backstage/config'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; @@ -36,12 +35,13 @@ import { HumanDuration } from '@backstage/types'; export interface PluginOptions { schedule?: TaskScheduleDefinition; age?: HumanDuration; + batchSize?: number; + useSourceLocation?: boolean; } /** @public */ export interface RouterOptions { linguistBackendApi?: LinguistBackendApi; - config: Config; logger: Logger; reader: UrlReader; database: PluginDatabaseManager; @@ -54,10 +54,9 @@ export async function createRouter( pluginOptions: PluginOptions, routerOptions: RouterOptions, ): Promise { - const { schedule, age } = pluginOptions; + const { schedule, age, batchSize, useSourceLocation } = pluginOptions; - const { config, logger, reader, database, discovery, scheduler } = - routerOptions; + const { logger, reader, database, discovery, scheduler } = routerOptions; const linguistBackendStore = await LinguistBackendDatabase.create( await database.getClient(), @@ -66,12 +65,13 @@ export async function createRouter( const linguistBackendApi = routerOptions.linguistBackendApi || new LinguistBackendApi( - config, logger, linguistBackendStore, reader, discovery, age, + batchSize, + useSourceLocation, ); if (scheduler && schedule) { diff --git a/plugins/linguist-backend/src/service/standaloneServer.ts b/plugins/linguist-backend/src/service/standaloneServer.ts index 874806e788..828c83630a 100644 --- a/plugins/linguist-backend/src/service/standaloneServer.ts +++ b/plugins/linguist-backend/src/service/standaloneServer.ts @@ -60,10 +60,9 @@ export async function startStandaloneServer( logger.debug('Starting application server...'); const router = await createRouter( - { schedule: schedule, age: { days: 30 } }, + { schedule: schedule, age: { days: 30 }, useSourceLocation: false }, { database: { getClient: async () => db }, - config, discovery: SingleHostDiscovery.fromConfig(config), reader: UrlReaders.default({ logger, config }), logger, diff --git a/plugins/linguist-common/api-report.md b/plugins/linguist-common/api-report.md index a9eceb3839..954bef7904 100644 --- a/plugins/linguist-common/api-report.md +++ b/plugins/linguist-common/api-report.md @@ -3,15 +3,13 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { Entity } from '@backstage/catalog-model'; - // @public (undocumented) export type EntitiesOverview = { entityCount: number; processedCount: number; pendingCount: number; staleCount: number; - filteredEntities: Entity[]; + filteredEntities: string[]; }; // @public (undocumented) @@ -43,7 +41,7 @@ export const LINGUIST_ANNOTATION = 'backstage.io/linguist'; // @public (undocumented) export type ProcessedEntity = { entityRef: string; - processed_date: Date; + processedDate: Date; }; // (No @packageDocumentation comment for this package) diff --git a/plugins/linguist-common/package.json b/plugins/linguist-common/package.json index eddc114a67..116f785216 100644 --- a/plugins/linguist-common/package.json +++ b/plugins/linguist-common/package.json @@ -22,10 +22,6 @@ "prepack": "backstage-cli package prepack", "postpack": "backstage-cli package postpack" }, - "dependencies": { - "@backstage/catalog-model": "workspace:^", - "@backstage/plugin-permission-common": "workspace:^" - }, "devDependencies": { "@backstage/cli": "workspace:^" }, diff --git a/plugins/linguist-common/src/types.ts b/plugins/linguist-common/src/types.ts index 01222492b6..28a25ed059 100644 --- a/plugins/linguist-common/src/types.ts +++ b/plugins/linguist-common/src/types.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; /** @public */ export type EntityResults = { @@ -41,7 +40,7 @@ export type Language = { /** @public */ export type ProcessedEntity = { entityRef: string; - processed_date: Date; + processedDate: Date; }; /** @public */ @@ -50,5 +49,5 @@ export type EntitiesOverview = { processedCount: number; pendingCount: number; staleCount: number; - filteredEntities: Entity[]; + filteredEntities: string[]; }; diff --git a/yarn.lock b/yarn.lock index 9dd67caa8b..4120623904 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6930,9 +6930,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-linguist-common@workspace:plugins/linguist-common" dependencies: - "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" - "@backstage/plugin-permission-common": "workspace:^" languageName: unknown linkType: soft