paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 34 Packages Projects Releases Wiki Activity

Refactored processing

Browse Source 
Signed-off-by: Andre Wanlin <andrewanlin@gmail.com>
This commit is contained in:
Andre Wanlin
2023-02-09 14:58:01 -06:00
parent d327896ecb
commit 54f08f8bd9
13 changed files with 199 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend/src/plugins/linguist.ts
+9 -1
View File
@@ -28,5 +28,13 @@ export default async function createPlugin(
initialDelay: { seconds: 15 },
};
return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env });
return createRouter(
{
schedule: schedule,
age: { days: 30 },
batchSize: 2,
useSourceLocation: false,
},
{ ...env },
);
}
plugins/linguist-backend/README.md
+22 -11
View File
@@ -31,8 +31,8 @@ Here's how to get the backend up and running:
env: PluginEnvironment,
): Promise<Router> {
const schedule: TaskScheduleDefinition = {
frequency: { hours: 24 },
timeout: { minutes: 30 },
frequency: { minutes: 2 },
timeout: { minutes: 15 },
initialDelay: { seconds: 90 },
};
@@ -57,20 +57,18 @@ Here's how to get the backend up and running:
4. Now run `yarn start-backend` from the repo root
5. Finally open `http://localhost:7007/api/linguist/health` in a browser and it should return `{"status":"ok"}`
## Scheduling
## Batch Size
The Linguist backend can be configured to generate the language breakdown for all the entities with the linguist annotation. This is done by passing a `TaskScheduleDefinition` to the `createRouter` function like in Step 2 of the [Up and Running](#up-and-running) documentation.
Here's another example of the `TaskScheduleDefinition` that will run the language breakdown processing every 12 hours:
The Linguist backend is setup to process entities by acting as a queue where it will pull down all the applicable entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest entities that have not been processed to determine their languages and process them. To control the batch size simply provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this:
```ts
const schedule: TaskScheduleDefinition = {
frequency: { hours: 12 },
timeout: { minutes: 15 },
initialDelay: { seconds: 10 },
};
return createRouter({ schedule: schedule, batchSize: 40 }, { ...env });
```
**Note:** The default batch size is 20
## Refresh
The default setup will only generate the language breakdown for entities with the linguist annotation that have not been generated yet. If you want this process to also refresh the data you can do so by adding the `age` (as a `HumanDuration`) in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`:
```ts
@@ -79,6 +77,19 @@ return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env });
With the `age` setup like this if the language breakdown is older than 15 days it will get regenerated. It's recommended that if you choose to use this configuration to set it to a large value - 30, 90, or 180 - as this data generally does not change drastically.
## Use Source Location
You may wish to use the `backstage.io/source-location` annotation over using the `backstage.io/linguist` as you may not be able to quickly add that annotation to your Entities. To do this you'll just need to set the `useSourceLocation` boolean to `true` in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`:
```ts
return createRouter(
{ schedule: schedule, useSourceLocation: true },
{ ...env },
);
```
**Note:** This has the potential to cause a lot of processing, be very thoughtful about this before hand
## Links
- [Frontend part of the plugin](../linguist/README.md)
plugins/linguist-backend/api-report.md
+17 -7
View File
@@ -3,7 +3,6 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Config } from '@backstage/config';
import { EntitiesOverview } from '@backstage/plugin-linguist-common';
import { EntityResults } from '@backstage/plugin-linguist-common';
import express from 'express';
@@ -27,12 +26,13 @@ export function createRouter(
// @public (undocumented)
export class LinguistBackendApi {
constructor(
config: Config,
logger: Logger,
store: LinguistBackendStore,
urlReader: UrlReader,
discovery: PluginEndpointDiscovery,
age?: HumanDuration | undefined,
age?: HumanDuration,
batchSize?: number,
useSourceLocation?: boolean,
);
// (undocumented)
getEntitiesOverview(): Promise<EntitiesOverview>;
@@ -52,9 +52,13 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
// (undocumented)
getEntityResults(entityRef: string): Promise<Languages>;
// (undocumented)
getProcessedEntities(): Promise<ProcessedEntity[]>;
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
// (undocumented)
getUnprocessedEntities(): Promise<string[] | []>;
// (undocumented)
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
// (undocumented)
insertNewEntity(entityRef: string): Promise<void>;
}
// @public (undocumented)
@@ -62,9 +66,13 @@ export interface LinguistBackendStore {
// (undocumented)
getEntityResults(entityRef: string): Promise<Languages>;
// (undocumented)
getProcessedEntities(): Promise<ProcessedEntity[]>;
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
// (undocumented)
getUnprocessedEntities(): Promise<string[] | []>;
// (undocumented)
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
// (undocumented)
insertNewEntity(entityRef: string): Promise<void>;
}
// @public (undocumented)
@@ -72,13 +80,15 @@ export interface PluginOptions {
// (undocumented)
age?: HumanDuration;
// (undocumented)
batchSize?: number;
// (undocumented)
schedule?: TaskScheduleDefinition;
// (undocumented)
useSourceLocation?: boolean;
}
// @public (undocumented)
export interface RouterOptions {
// (undocumented)
config: Config;
// (undocumented)
database: PluginDatabaseManager;
// (undocumented)
plugins/linguist-backend/migrations/20221115_init.js
+1 -5
View File
@@ -36,10 +36,7 @@ exports.up = async function up(knex) {
.unique()
.notNullable()
.comment('The entity ref that this Linguist result applies to');
table
.text('languages')
.notNullable()
.comment('The results json as a string');
table.text('languages').comment('The results json as a string');
table
.dateTime('created_at')
.defaultTo(knex.fn.now())
@@ -48,7 +45,6 @@ exports.up = async function up(knex) {
table
.dateTime('processed_date')
.defaultTo(knex.fn.now())
.notNullable()
.comment('The timestamp when this entity was processed');
table.index('index', 'entity_result_index_idx');
table.index('entity_ref', 'entity_result_entity_ref_idx');
plugins/linguist-backend/src/api/LinguistBackendApi.ts
+88 -52
View File
@@ -27,27 +27,48 @@ import {
} from '@backstage/catalog-client';
import { PluginEndpointDiscovery, UrlReader } from '@backstage/backend-common';
import { Config } from '@backstage/config';
import { DateTime } from 'luxon';
import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common';
import { LinguistBackendStore } from '../db';
import { Logger } from 'winston';
import fs from 'fs-extra';
import linguist from 'linguist-js';
import { stringifyEntityRef } from '@backstage/catalog-model';
import {
ANNOTATION_SOURCE_LOCATION,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { assertError } from '@backstage/errors';
import { HumanDuration } from '@backstage/types';
/** @public */
export class LinguistBackendApi {
private readonly logger: Logger;
private readonly store: LinguistBackendStore;
private readonly urlReader: UrlReader;
private readonly discovery: PluginEndpointDiscovery;
private readonly catalogClient: CatalogClient;
private readonly age?: HumanDuration;
private readonly batchSize?: number;
private readonly useSourceLocation?: boolean;
public constructor(
private readonly config: Config,
private readonly logger: Logger,
private readonly store: LinguistBackendStore,
private readonly urlReader: UrlReader,
private readonly discovery: PluginEndpointDiscovery,
private readonly age?: HumanDuration,
) {}
logger: Logger,
store: LinguistBackendStore,
urlReader: UrlReader,
discovery: PluginEndpointDiscovery,
age?: HumanDuration,
batchSize?: number,
useSourceLocation?: boolean,
) {
this.logger = logger;
this.store = store;
this.urlReader = urlReader;
this.discovery = discovery;
this.catalogClient = new CatalogClient({ discoveryApi: this.discovery });
this.batchSize = batchSize;
this.age = age;
this.useSourceLocation = useSourceLocation;
}
public async getEntityLanguages(entityRef: string): Promise<Languages> {
this.logger?.debug(`Getting languages for entity "${entityRef}"`);
@@ -105,65 +126,59 @@ export class LinguistBackendApi {
}
public async processEntities() {
this.logger?.info('Updating list of entities');
await this.addNewEntities();
this.logger?.info('Processing applicable entities through Linguist');
const eo = await this.getEntitiesOverview();
const entitiesOverview = await this.getEntitiesOverview();
this.logger?.info(
`Entities overview: Entity: ${eo.entityCount}, Processed: ${eo.processedCount}, Pending: ${eo.pendingCount}, Stale ${eo.staleCount}`,
`Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`,
);
const entities = eo.filteredEntities;
for (const key in entities) {
if (Object.prototype.hasOwnProperty.call(entities, key)) {
const entityRef = stringifyEntityRef(entities[key]);
const entities = entitiesOverview.filteredEntities.slice(
this.batchSize ?? 20,
);
entities.forEach(async entityRef => {
const entity = await this.catalogClient.getEntityByRef(entityRef);
const annotationKey = this.useSourceLocation
? ANNOTATION_SOURCE_LOCATION
: LINGUIST_ANNOTATION;
const url =
entities[key].metadata.annotations?.[LINGUIST_ANNOTATION] ?? '';
try {
await this.processEntity(entityRef, url);
} catch (e) {
assertError(e);
this.logger.error(
`Unable to process "${entityRef}" using "${url}", ${e.message}`,
);
}
let url = entity?.metadata.annotations?.[annotationKey] ?? '';
if (url.startsWith('url:')) {
url = url.slice(4);
}
}
try {
await this.processEntity(entityRef, url);
} catch (error) {
assertError(error);
this.logger.error(
`Unable to process "${entityRef}" using "${url}", message: ${error.message}, stack: ${error.stack}`,
);
}
});
}
public async getEntitiesOverview(): Promise<EntitiesOverview> {
this.logger?.debug('Getting pending entities');
const age = this.config.getOptionalNumber('linguist.age') ?? 0;
const catalogApi = new CatalogClient({ discoveryApi: this.discovery });
const request: GetEntitiesRequest = {
filter: {
kind: ['API', 'Component', 'Template'],
[`metadata.annotations.${LINGUIST_ANNOTATION}`]: CATALOG_FILTER_EXISTS,
},
fields: ['kind', 'metadata'],
};
const response = await catalogApi.getEntities(request);
const entities = response.items;
const processedEntities = await this.store.getProcessedEntities();
const staleEntities = processedEntities
.filter(pe => {
if (this.age === undefined) return false;
const staleDate = DateTime.now().minus(this.age as HumanDuration);
return DateTime.fromJSDate(pe.processedDate) <= staleDate;
})
.map(pe => pe.entityRef);
const staleEntities = processedEntities.filter(pe => {
if (age === undefined) return false;
const staleDate = DateTime.now().minus(this.age as HumanDuration);
return DateTime.fromJSDate(pe.processed_date) >= staleDate;
});
const filteredEntities = entities.filter(en => {
return processedEntities.every(pe => {
const entityRef = stringifyEntityRef(en);
return pe.entityRef !== entityRef;
});
});
const unprocessedEntities = await this.store.getUnprocessedEntities();
const filteredEntities = staleEntities.concat(unprocessedEntities);
const entitiesOverview: EntitiesOverview = {
entityCount: entities.length,
entityCount: unprocessedEntities.length,
processedCount: processedEntities.length,
staleCount: staleEntities.length,
pendingCount: filteredEntities.length,
@@ -172,4 +187,25 @@ export class LinguistBackendApi {
return entitiesOverview;
}
private async addNewEntities() {
const annotationKey = this.useSourceLocation
? ANNOTATION_SOURCE_LOCATION
: LINGUIST_ANNOTATION;
const request: GetEntitiesRequest = {
filter: {
kind: ['API', 'Component', 'Template'],
[`metadata.annotations.${annotationKey}`]: CATALOG_FILTER_EXISTS,
},
fields: ['kind', 'metadata'],
};
const response = await this.catalogClient.getEntities(request);
const entities = response.items;
entities.forEach(entity => {
const entityRef = stringifyEntityRef(entity);
this.store.insertNewEntity(entityRef);
});
}
}
plugins/linguist-backend/src/db/LinguistBackendDatabase.ts
+50 -6
View File
@@ -33,8 +33,10 @@ export type RawDbEntityResultRow = {
/** @public */
export interface LinguistBackendStore {
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
insertNewEntity(entityRef: string): Promise<void>;
getEntityResults(entityRef: string): Promise<Languages>;
getProcessedEntities(): Promise<ProcessedEntity[]>;
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
getUnprocessedEntities(): Promise<string[] | []>;
}
const migrationsDir = resolvePackagePath(
@@ -71,8 +73,21 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
return result.id;
}
async insertNewEntity(entityRef: string): Promise<void> {
const entityLanguageId = uuid();
await this.db<RawDbEntityResultRow>('entity_result')
.insert({
id: entityLanguageId,
entity_ref: entityRef,
})
.onConflict('entity_ref')
.ignore(); // If the entity_ref is in the table already then we don't want to add it again
}
async getEntityResults(entityRef: string): Promise<Languages> {
const entityResults = await this.db<RawDbEntityResultRow>('entity_result')
.whereNotNull('languages')
.where({ entity_ref: entityRef })
.first();
@@ -93,11 +108,40 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
}
}
async getProcessedEntities(): Promise<ProcessedEntity[]> {
const entityResults = await this.db<ProcessedEntity>(
'entity_result',
).select('entity_ref', 'processed_date');
async getProcessedEntities(): Promise<ProcessedEntity[] | []> {
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result')
.whereNotNull('processed_date')
.whereNotNull('languages');
return entityResults;
if (!rawEntities) {
return [];
}
const processedEntities = rawEntities.map(rawEntity => {
const processEntity = {
entityRef: rawEntity.entity_ref,
processedDate: rawEntity.processed_date,
};
return processEntity;
});
return processedEntities;
}
async getUnprocessedEntities(): Promise<string[] | []> {
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result')
.whereNull('languages')
.orderBy('created_at', 'asc');
if (!rawEntities) {
return [];
}
const unprocessedEntities = rawEntities.map(rawEntity => {
return rawEntity.entity_ref;
});
return unprocessedEntities;
}
}
plugins/linguist-backend/src/service/router.test.ts
+1 -2
View File
@@ -65,10 +65,9 @@ describe('createRouter', () => {
beforeAll(async () => {
const router = await createRouter(
{ schedule: schedule, age: { days: 30 } },
{ schedule: schedule, age: { days: 30 }, useSourceLocation: false },
{
linguistBackendApi,
config: new ConfigReader({}),
discovery: testDiscovery,
database: createDatabase(),
reader: mockUrlReader,
plugins/linguist-backend/src/service/router.ts
+6 -6
View File
@@ -20,7 +20,6 @@ import {
PluginEndpointDiscovery,
UrlReader,
} from '@backstage/backend-common';
import { Config } from '@backstage/config';
import express from 'express';
import Router from 'express-promise-router';
import { Logger } from 'winston';
@@ -36,12 +35,13 @@ import { HumanDuration } from '@backstage/types';
export interface PluginOptions {
schedule?: TaskScheduleDefinition;
age?: HumanDuration;
batchSize?: number;
useSourceLocation?: boolean;
}
/** @public */
export interface RouterOptions {
linguistBackendApi?: LinguistBackendApi;
config: Config;
logger: Logger;
reader: UrlReader;
database: PluginDatabaseManager;
@@ -54,10 +54,9 @@ export async function createRouter(
pluginOptions: PluginOptions,
routerOptions: RouterOptions,
): Promise<express.Router> {
const { schedule, age } = pluginOptions;
const { schedule, age, batchSize, useSourceLocation } = pluginOptions;
const { config, logger, reader, database, discovery, scheduler } =
routerOptions;
const { logger, reader, database, discovery, scheduler } = routerOptions;
const linguistBackendStore = await LinguistBackendDatabase.create(
await database.getClient(),
@@ -66,12 +65,13 @@ export async function createRouter(
const linguistBackendApi =
routerOptions.linguistBackendApi ||
new LinguistBackendApi(
config,
logger,
linguistBackendStore,
reader,
discovery,
age,
batchSize,
useSourceLocation,
);
if (scheduler && schedule) {
plugins/linguist-backend/src/service/standaloneServer.ts
+1 -2
View File
@@ -60,10 +60,9 @@ export async function startStandaloneServer(
logger.debug('Starting application server...');
const router = await createRouter(
{ schedule: schedule, age: { days: 30 } },
{ schedule: schedule, age: { days: 30 }, useSourceLocation: false },
{
database: { getClient: async () => db },
config,
discovery: SingleHostDiscovery.fromConfig(config),
reader: UrlReaders.default({ logger, config }),
logger,
plugins/linguist-common/api-report.md
+2 -4
View File
@@ -3,15 +3,13 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { Entity } from '@backstage/catalog-model';
// @public (undocumented)
export type EntitiesOverview = {
entityCount: number;
processedCount: number;
pendingCount: number;
staleCount: number;
filteredEntities: Entity[];
filteredEntities: string[];
};
// @public (undocumented)
@@ -43,7 +41,7 @@ export const LINGUIST_ANNOTATION = 'backstage.io/linguist';
// @public (undocumented)
export type ProcessedEntity = {
entityRef: string;
processed_date: Date;
processedDate: Date;
};
// (No @packageDocumentation comment for this package)
plugins/linguist-common/package.json
-4
View File
@@ -22,10 +22,6 @@
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/catalog-model": "workspace:^",
"@backstage/plugin-permission-common": "workspace:^"
},
"devDependencies": {
"@backstage/cli": "workspace:^"
},
plugins/linguist-common/src/types.ts
+2 -3
View File
@@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity } from '@backstage/catalog-model';
/** @public */
export type EntityResults = {
@@ -41,7 +40,7 @@ export type Language = {
/** @public */
export type ProcessedEntity = {
entityRef: string;
processed_date: Date;
processedDate: Date;
};
/** @public */
@@ -50,5 +49,5 @@ export type EntitiesOverview = {
processedCount: number;
pendingCount: number;
staleCount: number;
filteredEntities: Entity[];
filteredEntities: string[];
};
yarn.lock
-2
View File
@@ -6930,9 +6930,7 @@ __metadata:
version: 0.0.0-use.local
resolution: "@backstage/plugin-linguist-common@workspace:plugins/linguist-common"
dependencies:
"@backstage/catalog-model": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/plugin-permission-common": "workspace:^"
languageName: unknown
linkType: soft