Merge branch 'master' of github.com:codingdiaz/backstage into feat/tools-collator-auth
This commit is contained in:
Christopher Diaz
@@ -57,7 +57,11 @@ Here's how to get the backend up and running:
|
||||
4. Now run `yarn start-backend` from the repo root
|
||||
5. Finally open `http://localhost:7007/api/linguist/health` in a browser and it should return `{"status":"ok"}`
|
||||
|
||||
## Batch Size
|
||||
## Plugin Option
|
||||
|
||||
The Linguist backend has various plugin options that you can provide to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` file that will allow you to configure various aspects of how it works. The following sections go into the details of these options
|
||||
|
||||
### Batch Size
|
||||
|
||||
The Linguist backend is setup to process entities by acting as a queue where it will pull down all the applicable entities from the Catalog and add them to it's database (saving just the `entityRef`). Then it will grab the `n` oldest entities that have not been processed to determine their languages and process them. To control the batch size simply provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this:
|
||||
|
||||
@@ -67,7 +71,7 @@ return createRouter({ schedule: schedule, batchSize: 40 }, { ...env });
|
||||
|
||||
**Note:** The default batch size is 20
|
||||
|
||||
## Kind
|
||||
### Kind
|
||||
|
||||
The default setup only processes entities of kind `['API', 'Component', 'Template']`. To control the `kind` that are processed provide that to the `createRouter` function in your `packages/backend/src/plugins/linguist.ts` like this:
|
||||
|
||||
@@ -75,7 +79,7 @@ The default setup only processes entities of kind `['API', 'Component', 'Templat
|
||||
return createRouter({ schedule: schedule, kind: ['Component'] }, { ...env });
|
||||
```
|
||||
|
||||
## Refresh
|
||||
### Refresh
|
||||
|
||||
The default setup will only generate the language breakdown for entities with the linguist annotation that have not been generated yet. If you want this process to also refresh the data you can do so by adding the `age` (as a `HumanDuration`) in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`:
|
||||
|
||||
@@ -85,7 +89,7 @@ return createRouter({ schedule: schedule, age: { days: 30 } }, { ...env });
|
||||
|
||||
With the `age` setup like this if the language breakdown is older than 15 days it will get regenerated. It's recommended that if you choose to use this configuration to set it to a large value - 30, 90, or 180 - as this data generally does not change drastically.
|
||||
|
||||
## Linguist JS options
|
||||
### Linguist JS options
|
||||
|
||||
The default setup will use the default [linguist-js](https://www.npmjs.com/package/linguist-js) options, a full list of the available options can be found [here](https://www.npmjs.com/package/linguist-js#API).
|
||||
|
||||
@@ -96,7 +100,7 @@ return createRouter(
|
||||
);
|
||||
```
|
||||
|
||||
## Use Source Location
|
||||
### Use Source Location
|
||||
|
||||
You may wish to use the `backstage.io/source-location` annotation over using the `backstage.io/linguist` as you may not be able to quickly add that annotation to your Entities. To do this you'll just need to set the `useSourceLocation` boolean to `true` in your `packages/backend/src/plugins/linguist.ts` when you call `createRouter`:
|
||||
|
||||
|
||||
@@ -3,16 +3,13 @@
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
import { EntityResults } from '@backstage/plugin-linguist-common';
|
||||
import express from 'express';
|
||||
import { HumanDuration } from '@backstage/types';
|
||||
import { Knex } from 'knex';
|
||||
import { Languages } from '@backstage/plugin-linguist-common';
|
||||
import { Logger } from 'winston';
|
||||
import { PluginDatabaseManager } from '@backstage/backend-common';
|
||||
import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
import { PluginTaskScheduler } from '@backstage/backend-tasks';
|
||||
import { ProcessedEntity } from '@backstage/plugin-linguist-common';
|
||||
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
|
||||
import { TokenManager } from '@backstage/backend-common';
|
||||
import { UrlReader } from '@backstage/backend-common';
|
||||
@@ -24,56 +21,13 @@ export function createRouter(
|
||||
): Promise<express.Router>;
|
||||
|
||||
// @public (undocumented)
|
||||
export class LinguistBackendApi {
|
||||
constructor(
|
||||
logger: Logger,
|
||||
store: LinguistBackendStore,
|
||||
urlReader: UrlReader,
|
||||
discovery: PluginEndpointDiscovery,
|
||||
tokenManager: TokenManager,
|
||||
age?: HumanDuration,
|
||||
batchSize?: number,
|
||||
useSourceLocation?: boolean,
|
||||
kind?: string[],
|
||||
linguistJsOptions?: Record<string, unknown>,
|
||||
);
|
||||
export interface LinguistBackendApi {
|
||||
// (undocumented)
|
||||
getEntityLanguages(entityRef: string): Promise<Languages>;
|
||||
// (undocumented)
|
||||
processEntities(): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
constructor(db: Knex);
|
||||
// (undocumented)
|
||||
static create(knex: Knex): Promise<LinguistBackendStore>;
|
||||
// (undocumented)
|
||||
getEntityResults(entityRef: string): Promise<Languages>;
|
||||
// (undocumented)
|
||||
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
|
||||
// (undocumented)
|
||||
getUnprocessedEntities(): Promise<string[] | []>;
|
||||
// (undocumented)
|
||||
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
|
||||
// (undocumented)
|
||||
insertNewEntity(entityRef: string): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface LinguistBackendStore {
|
||||
// (undocumented)
|
||||
getEntityResults(entityRef: string): Promise<Languages>;
|
||||
// (undocumented)
|
||||
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
|
||||
// (undocumented)
|
||||
getUnprocessedEntities(): Promise<string[] | []>;
|
||||
// (undocumented)
|
||||
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
|
||||
// (undocumented)
|
||||
insertNewEntity(entityRef: string): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PluginOptions {
|
||||
// (undocumented)
|
||||
|
||||
+23
-12
@@ -13,16 +13,27 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { kindOrDefault } from './LinguistBackendApi';
|
||||
|
||||
describe('kindOrDefault', () => {
|
||||
it('should return default kind when undefined', () => {
|
||||
expect(kindOrDefault()).toEqual(['API', 'Component', 'Template']);
|
||||
});
|
||||
it('should return the default kind when empty', () => {
|
||||
expect(kindOrDefault([])).toEqual(['API', 'Component', 'Template']);
|
||||
});
|
||||
it('should return provided kind when not empty', () => {
|
||||
expect(kindOrDefault(['API'])).toEqual(['API']);
|
||||
});
|
||||
});
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.up = async function up(knex) {
|
||||
// Sqlite does not support this raw SQL
|
||||
if (!knex.client.config.client.includes('sqlite3')) {
|
||||
await knex.raw(
|
||||
'ALTER TABLE entity_result ALTER COLUMN processed_date DROP DEFAULT;',
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* @param {import('knex').Knex} knex
|
||||
*/
|
||||
exports.down = async function down(knex) {
|
||||
// Sqlite does not support this raw SQL
|
||||
if (!knex.client.config.client.includes('sqlite3')) {
|
||||
await knex.raw(
|
||||
'ALTER TABLE entity_result ALTER COLUMN processed_date SET DEFAULT now();',
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -44,6 +44,7 @@
|
||||
"yn": "^4.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@backstage/backend-test-utils": "workspace:^",
|
||||
"@backstage/cli": "workspace:^",
|
||||
"@types/supertest": "^2.0.8",
|
||||
"msw": "^1.0.0",
|
||||
|
||||
@@ -0,0 +1,416 @@
|
||||
/*
|
||||
* Copyright 2023 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {
|
||||
getVoidLogger,
|
||||
ReadTreeResponse,
|
||||
ServerTokenManager,
|
||||
UrlReader,
|
||||
} from '@backstage/backend-common';
|
||||
import { CatalogApi, GetEntitiesResponse } from '@backstage/catalog-client';
|
||||
import { Results } from 'linguist-js/dist/types';
|
||||
import { DateTime } from 'luxon';
|
||||
import { LinguistBackendStore } from '../db';
|
||||
import { kindOrDefault, LinguistBackendClient } from './LinguistBackendClient';
|
||||
import fs from 'fs-extra';
|
||||
import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common';
|
||||
|
||||
const linguistResultMock = Promise.resolve({
|
||||
files: {
|
||||
count: 4,
|
||||
bytes: 6010,
|
||||
results: {
|
||||
'/src/index.ts': 'TypeScript',
|
||||
'/src/cli.js': 'JavaScript',
|
||||
'/readme.md': 'Markdown',
|
||||
'/no-lang': null,
|
||||
},
|
||||
},
|
||||
languages: {
|
||||
count: 3,
|
||||
bytes: 6000,
|
||||
results: {
|
||||
JavaScript: { type: 'programming', bytes: 1000, color: '#f1e05a' },
|
||||
TypeScript: { type: 'programming', bytes: 2000, color: '#2b7489' },
|
||||
Markdown: { type: 'prose', bytes: 3000, color: '#083fa1' },
|
||||
},
|
||||
},
|
||||
unknown: {
|
||||
count: 1,
|
||||
bytes: 10,
|
||||
filenames: {
|
||||
'no-lang': 10,
|
||||
},
|
||||
extensions: {},
|
||||
},
|
||||
} as Results);
|
||||
|
||||
describe('kindOrDefault', () => {
|
||||
it('should return default kind when undefined', () => {
|
||||
expect(kindOrDefault()).toEqual(['API', 'Component', 'Template']);
|
||||
});
|
||||
it('should return the default kind when empty', () => {
|
||||
expect(kindOrDefault([])).toEqual(['API', 'Component', 'Template']);
|
||||
});
|
||||
it('should return provided kind when not empty', () => {
|
||||
expect(kindOrDefault(['API'])).toEqual(['API']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Linguist backend API', () => {
|
||||
const logger = getVoidLogger();
|
||||
|
||||
const store: jest.Mocked<LinguistBackendStore> = {
|
||||
insertEntityResults: jest.fn(),
|
||||
insertNewEntity: jest.fn(),
|
||||
getEntityResults: jest.fn(),
|
||||
getProcessedEntities: jest.fn(),
|
||||
getUnprocessedEntities: jest.fn(),
|
||||
getAllEntities: jest.fn(),
|
||||
deleteEntity: jest.fn(),
|
||||
};
|
||||
|
||||
const urlReader: jest.Mocked<UrlReader> = {
|
||||
readTree: jest.fn(),
|
||||
search: jest.fn(),
|
||||
readUrl: jest.fn(),
|
||||
};
|
||||
|
||||
const catalogApi: jest.Mocked<CatalogApi> = {
|
||||
getEntities: jest.fn(),
|
||||
getEntityByRef: jest.fn(),
|
||||
} as any;
|
||||
|
||||
const tokenManager = ServerTokenManager.noop();
|
||||
|
||||
const api = new LinguistBackendClient(
|
||||
logger,
|
||||
store,
|
||||
urlReader,
|
||||
tokenManager,
|
||||
catalogApi,
|
||||
);
|
||||
|
||||
beforeEach(() => {
|
||||
jest.resetAllMocks();
|
||||
});
|
||||
|
||||
it('should get languages for an entity', async () => {
|
||||
store.getEntityResults.mockResolvedValue({
|
||||
languageCount: 1,
|
||||
totalBytes: 2205,
|
||||
processedDate: '2023-02-15T20:10:21.378Z',
|
||||
breakdown: [
|
||||
{
|
||||
name: 'YAML',
|
||||
percentage: 100,
|
||||
bytes: 2205,
|
||||
type: 'data',
|
||||
color: '#cb171e',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const entityRef = 'template:default/create-react-app-template';
|
||||
const languages = await api.getEntityLanguages(entityRef);
|
||||
expect(languages).toEqual({
|
||||
languageCount: 1,
|
||||
totalBytes: 2205,
|
||||
processedDate: '2023-02-15T20:10:21.378Z',
|
||||
breakdown: [
|
||||
{
|
||||
name: 'YAML',
|
||||
percentage: 100,
|
||||
bytes: 2205,
|
||||
type: 'data',
|
||||
color: '#cb171e',
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('should insert new entities', async () => {
|
||||
const testEntityListResponse: GetEntitiesResponse = {
|
||||
items: [
|
||||
{
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-one',
|
||||
},
|
||||
kind: 'Component',
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-two',
|
||||
},
|
||||
kind: 'Component',
|
||||
},
|
||||
{
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-three',
|
||||
},
|
||||
kind: 'Component',
|
||||
},
|
||||
],
|
||||
};
|
||||
catalogApi.getEntities.mockResolvedValue(testEntityListResponse);
|
||||
|
||||
await api.addNewEntities();
|
||||
expect(store.insertNewEntity).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('should delete entities not in Catalog', async () => {
|
||||
store.getAllEntities.mockResolvedValue([
|
||||
'component:default/service-one',
|
||||
'component:default/stale-service-two',
|
||||
]);
|
||||
|
||||
catalogApi.getEntityByRef.mockResolvedValueOnce({
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-one',
|
||||
},
|
||||
kind: 'Component',
|
||||
});
|
||||
|
||||
await api.cleanEntities();
|
||||
expect(store.deleteEntity).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('should get default entity overview', async () => {
|
||||
store.getProcessedEntities.mockResolvedValue([
|
||||
{
|
||||
entityRef: 'component:default/service-one',
|
||||
processedDate: DateTime.now().toJSDate(),
|
||||
},
|
||||
{
|
||||
entityRef: 'component:default/stale-service-two',
|
||||
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
|
||||
},
|
||||
]);
|
||||
|
||||
store.getUnprocessedEntities.mockResolvedValue([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
]);
|
||||
|
||||
const overview = await api.getEntitiesOverview();
|
||||
expect(overview.entityCount).toEqual(5);
|
||||
expect(overview.processedCount).toEqual(2);
|
||||
expect(overview.staleCount).toEqual(0);
|
||||
expect(overview.pendingCount).toEqual(3);
|
||||
expect(overview.filteredEntities).toEqual([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should get entity overview with stale items', async () => {
|
||||
const apiWithAge = new LinguistBackendClient(
|
||||
logger,
|
||||
store,
|
||||
urlReader,
|
||||
tokenManager,
|
||||
catalogApi,
|
||||
{ days: 5 },
|
||||
);
|
||||
store.getProcessedEntities.mockResolvedValue([
|
||||
{
|
||||
entityRef: 'component:default/service-one',
|
||||
processedDate: DateTime.now().toJSDate(),
|
||||
},
|
||||
{
|
||||
entityRef: 'component:default/stale-service-two',
|
||||
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
|
||||
},
|
||||
]);
|
||||
|
||||
store.getUnprocessedEntities.mockResolvedValue([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
]);
|
||||
|
||||
const overview = await apiWithAge.getEntitiesOverview();
|
||||
expect(overview.entityCount).toEqual(5);
|
||||
expect(overview.processedCount).toEqual(2);
|
||||
expect(overview.staleCount).toEqual(1);
|
||||
expect(overview.pendingCount).toEqual(4);
|
||||
expect(overview.filteredEntities).toEqual([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
'component:default/stale-service-two',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should generate and save languages for an entity', async () => {
|
||||
const spy = jest
|
||||
.spyOn(api, 'getLinguistResults')
|
||||
.mockImplementation(() => linguistResultMock);
|
||||
|
||||
urlReader.readTree.mockResolvedValueOnce({
|
||||
files: async () => [
|
||||
{
|
||||
content: async () => Buffer.from('-- XXX: code-data', 'utf8'),
|
||||
path: 'my-file.js',
|
||||
},
|
||||
],
|
||||
dir: async () => '/temp/my-code',
|
||||
} as ReadTreeResponse);
|
||||
|
||||
const fsSpy = jest.spyOn(fs, 'remove');
|
||||
|
||||
await api.generateEntityLanguages(
|
||||
'component:default/fake-service',
|
||||
'https://some.fake/service/',
|
||||
);
|
||||
expect(api.getLinguistResults).toHaveBeenCalled();
|
||||
expect(store.insertEntityResults).toHaveBeenCalled();
|
||||
expect(fs.remove).toHaveBeenCalled();
|
||||
spy.mockClear();
|
||||
fsSpy.mockClear();
|
||||
});
|
||||
|
||||
it('should generate languages for entities using default', async () => {
|
||||
store.getProcessedEntities.mockResolvedValue([
|
||||
{
|
||||
entityRef: 'component:default/service-one',
|
||||
processedDate: DateTime.now().toJSDate(),
|
||||
},
|
||||
{
|
||||
entityRef: 'component:default/stale-service-two',
|
||||
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
|
||||
},
|
||||
]);
|
||||
|
||||
store.getUnprocessedEntities.mockResolvedValue([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
]);
|
||||
|
||||
const entity = {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-one',
|
||||
annotations: {
|
||||
[LINGUIST_ANNOTATION]: 'https://some.fake/service/',
|
||||
},
|
||||
},
|
||||
kind: 'Component',
|
||||
};
|
||||
|
||||
catalogApi.getEntityByRef.mockResolvedValue(entity);
|
||||
|
||||
const resultsSpy = jest
|
||||
.spyOn(api, 'getLinguistResults')
|
||||
.mockImplementation(() => linguistResultMock);
|
||||
|
||||
urlReader.readTree.mockResolvedValue({
|
||||
files: async () => [
|
||||
{
|
||||
content: async () => Buffer.from('-- XXX: code-data', 'utf8'),
|
||||
path: 'my-file.js',
|
||||
},
|
||||
],
|
||||
dir: async () => '/temp/my-code',
|
||||
} as ReadTreeResponse);
|
||||
|
||||
const fsSpy = jest.spyOn(fs, 'remove');
|
||||
|
||||
const generateEntityLanguages = jest.spyOn(api, 'generateEntityLanguages');
|
||||
await api.generateEntitiesLanguages();
|
||||
expect(generateEntityLanguages).toHaveBeenCalledTimes(3);
|
||||
|
||||
generateEntityLanguages.mockClear();
|
||||
resultsSpy.mockClear();
|
||||
fsSpy.mockClear();
|
||||
});
|
||||
|
||||
it('should generate languages for entities using defined batch size', async () => {
|
||||
const apiWithBatchSize = new LinguistBackendClient(
|
||||
logger,
|
||||
store,
|
||||
urlReader,
|
||||
tokenManager,
|
||||
catalogApi,
|
||||
undefined,
|
||||
2,
|
||||
);
|
||||
|
||||
store.getProcessedEntities.mockResolvedValue([
|
||||
{
|
||||
entityRef: 'component:default/service-one',
|
||||
processedDate: DateTime.now().toJSDate(),
|
||||
},
|
||||
{
|
||||
entityRef: 'component:default/stale-service-two',
|
||||
processedDate: DateTime.now().minus({ days: 45 }).toJSDate(),
|
||||
},
|
||||
]);
|
||||
|
||||
store.getUnprocessedEntities.mockResolvedValue([
|
||||
'component:default/service-three',
|
||||
'component:default/service-four',
|
||||
'component:default/service-five',
|
||||
]);
|
||||
|
||||
const entity = {
|
||||
apiVersion: 'backstage.io/v1beta1',
|
||||
metadata: {
|
||||
name: 'service-one',
|
||||
annotations: {
|
||||
[LINGUIST_ANNOTATION]: 'https://some.fake/service/',
|
||||
},
|
||||
},
|
||||
kind: 'Component',
|
||||
};
|
||||
|
||||
catalogApi.getEntityByRef.mockResolvedValue(entity);
|
||||
|
||||
const resultsSpy = jest
|
||||
.spyOn(apiWithBatchSize, 'getLinguistResults')
|
||||
.mockImplementation(() => linguistResultMock);
|
||||
|
||||
urlReader.readTree.mockResolvedValue({
|
||||
files: async () => [
|
||||
{
|
||||
content: async () => Buffer.from('-- XXX: code-data', 'utf8'),
|
||||
path: 'my-file.js',
|
||||
},
|
||||
],
|
||||
dir: async () => '/temp/my-code',
|
||||
} as ReadTreeResponse);
|
||||
|
||||
const fsSpy = jest.spyOn(fs, 'remove');
|
||||
|
||||
const generateEntityLanguages = jest.spyOn(
|
||||
apiWithBatchSize,
|
||||
'generateEntityLanguages',
|
||||
);
|
||||
await apiWithBatchSize.generateEntitiesLanguages();
|
||||
expect(generateEntityLanguages).toHaveBeenCalledTimes(2);
|
||||
|
||||
generateEntityLanguages.mockClear();
|
||||
resultsSpy.mockClear();
|
||||
fsSpy.mockClear();
|
||||
});
|
||||
});
|
||||
+58
-28
@@ -22,14 +22,10 @@ import {
|
||||
} from '@backstage/plugin-linguist-common';
|
||||
import {
|
||||
CATALOG_FILTER_EXISTS,
|
||||
CatalogClient,
|
||||
GetEntitiesRequest,
|
||||
CatalogApi,
|
||||
} from '@backstage/catalog-client';
|
||||
import {
|
||||
PluginEndpointDiscovery,
|
||||
TokenManager,
|
||||
UrlReader,
|
||||
} from '@backstage/backend-common';
|
||||
import { TokenManager, UrlReader } from '@backstage/backend-common';
|
||||
|
||||
import { DateTime } from 'luxon';
|
||||
import { LINGUIST_ANNOTATION } from '@backstage/plugin-linguist-common';
|
||||
@@ -43,16 +39,22 @@ import {
|
||||
} from '@backstage/catalog-model';
|
||||
import { assertError } from '@backstage/errors';
|
||||
import { HumanDuration } from '@backstage/types';
|
||||
import { Results } from 'linguist-js/dist/types';
|
||||
|
||||
/** @public */
|
||||
export class LinguistBackendApi {
|
||||
export interface LinguistBackendApi {
|
||||
getEntityLanguages(entityRef: string): Promise<Languages>;
|
||||
processEntities(): Promise<void>;
|
||||
}
|
||||
|
||||
/** @public */
|
||||
export class LinguistBackendClient implements LinguistBackendApi {
|
||||
private readonly logger: Logger;
|
||||
private readonly store: LinguistBackendStore;
|
||||
private readonly urlReader: UrlReader;
|
||||
private readonly discovery: PluginEndpointDiscovery;
|
||||
private readonly tokenManager: TokenManager;
|
||||
|
||||
private readonly catalogClient: CatalogClient;
|
||||
private readonly catalogApi: CatalogApi;
|
||||
private readonly age?: HumanDuration;
|
||||
private readonly batchSize?: number;
|
||||
private readonly useSourceLocation?: boolean;
|
||||
@@ -62,8 +64,8 @@ export class LinguistBackendApi {
|
||||
logger: Logger,
|
||||
store: LinguistBackendStore,
|
||||
urlReader: UrlReader,
|
||||
discovery: PluginEndpointDiscovery,
|
||||
tokenManager: TokenManager,
|
||||
catalogApi: CatalogApi,
|
||||
age?: HumanDuration,
|
||||
batchSize?: number,
|
||||
useSourceLocation?: boolean,
|
||||
@@ -73,9 +75,8 @@ export class LinguistBackendApi {
|
||||
this.logger = logger;
|
||||
this.store = store;
|
||||
this.urlReader = urlReader;
|
||||
this.discovery = discovery;
|
||||
this.tokenManager = tokenManager;
|
||||
this.catalogClient = new CatalogClient({ discoveryApi: this.discovery });
|
||||
this.catalogApi = catalogApi;
|
||||
this.batchSize = batchSize;
|
||||
this.age = age;
|
||||
this.useSourceLocation = useSourceLocation;
|
||||
@@ -83,23 +84,25 @@ export class LinguistBackendApi {
|
||||
this.linguistJsOptions = linguistJsOptions;
|
||||
}
|
||||
|
||||
public async getEntityLanguages(entityRef: string): Promise<Languages> {
|
||||
async getEntityLanguages(entityRef: string): Promise<Languages> {
|
||||
this.logger?.debug(`Getting languages for entity "${entityRef}"`);
|
||||
|
||||
return this.store.getEntityResults(entityRef);
|
||||
}
|
||||
|
||||
public async processEntities() {
|
||||
async processEntities(): Promise<void> {
|
||||
this.logger?.info('Updating list of entities');
|
||||
|
||||
await this.addNewEntities();
|
||||
|
||||
this.logger?.info('Processing applicable entities through Linguist');
|
||||
this.logger?.info('Cleaning list of entities');
|
||||
await this.cleanEntities();
|
||||
|
||||
this.logger?.info('Processing applicable entities through Linguist');
|
||||
await this.generateEntitiesLanguages();
|
||||
}
|
||||
|
||||
private async addNewEntities() {
|
||||
/** @internal */
|
||||
async addNewEntities(): Promise<void> {
|
||||
const annotationKey = this.useSourceLocation
|
||||
? ANNOTATION_SOURCE_LOCATION
|
||||
: LINGUIST_ANNOTATION;
|
||||
@@ -112,7 +115,7 @@ export class LinguistBackendApi {
|
||||
};
|
||||
|
||||
const { token } = await this.tokenManager.getToken();
|
||||
const response = await this.catalogClient.getEntities(request, { token });
|
||||
const response = await this.catalogApi.getEntities(request, { token });
|
||||
const entities = response.items;
|
||||
|
||||
entities.forEach(entity => {
|
||||
@@ -121,7 +124,25 @@ export class LinguistBackendApi {
|
||||
});
|
||||
}
|
||||
|
||||
private async generateEntitiesLanguages() {
|
||||
/** @internal */
|
||||
async cleanEntities(): Promise<void> {
|
||||
this.logger?.info('Cleaning entities in Linguist queue');
|
||||
const allEntities = await this.store.getAllEntities();
|
||||
|
||||
for (const entityRef of allEntities) {
|
||||
const result = await this.catalogApi.getEntityByRef(entityRef);
|
||||
|
||||
if (!result) {
|
||||
this.logger?.info(
|
||||
`Entity ${entityRef} was not found in the Catalog, it will be deleted`,
|
||||
);
|
||||
await this.store.deleteEntity(entityRef);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
async generateEntitiesLanguages(): Promise<void> {
|
||||
const entitiesOverview = await this.getEntitiesOverview();
|
||||
this.logger?.info(
|
||||
`Entities overview: Entity: ${entitiesOverview.entityCount}, Processed: ${entitiesOverview.processedCount}, Pending: ${entitiesOverview.pendingCount}, Stale ${entitiesOverview.staleCount}`,
|
||||
@@ -131,9 +152,10 @@ export class LinguistBackendApi {
|
||||
0,
|
||||
this.batchSize ?? 20,
|
||||
);
|
||||
entities.forEach(async entityRef => {
|
||||
|
||||
for (const entityRef of entities) {
|
||||
const { token } = await this.tokenManager.getToken();
|
||||
const entity = await this.catalogClient.getEntityByRef(entityRef, {
|
||||
const entity = await this.catalogApi.getEntityByRef(entityRef, {
|
||||
token,
|
||||
});
|
||||
const annotationKey = this.useSourceLocation
|
||||
@@ -153,10 +175,11 @@ export class LinguistBackendApi {
|
||||
`Unable to process "${entityRef}" using "${url}", message: ${error.message}, stack: ${error.stack}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async getEntitiesOverview(): Promise<EntitiesOverview> {
|
||||
/** @internal */
|
||||
async getEntitiesOverview(): Promise<EntitiesOverview> {
|
||||
this.logger?.debug('Getting pending entities');
|
||||
|
||||
const processedEntities = await this.store.getProcessedEntities();
|
||||
@@ -169,10 +192,10 @@ export class LinguistBackendApi {
|
||||
.map(pe => pe.entityRef);
|
||||
|
||||
const unprocessedEntities = await this.store.getUnprocessedEntities();
|
||||
const filteredEntities = staleEntities.concat(unprocessedEntities);
|
||||
const filteredEntities = unprocessedEntities.concat(staleEntities);
|
||||
|
||||
const entitiesOverview: EntitiesOverview = {
|
||||
entityCount: unprocessedEntities.length,
|
||||
entityCount: unprocessedEntities.length + processedEntities.length,
|
||||
processedCount: processedEntities.length,
|
||||
staleCount: staleEntities.length,
|
||||
pendingCount: filteredEntities.length,
|
||||
@@ -182,7 +205,8 @@ export class LinguistBackendApi {
|
||||
return entitiesOverview;
|
||||
}
|
||||
|
||||
private async generateEntityLanguages(
|
||||
/** @internal */
|
||||
async generateEntityLanguages(
|
||||
entityRef: string,
|
||||
url: string,
|
||||
): Promise<string> {
|
||||
@@ -193,7 +217,7 @@ export class LinguistBackendApi {
|
||||
const readTreeResponse = await this.urlReader.readTree(url);
|
||||
const dir = await readTreeResponse.dir();
|
||||
|
||||
const results = await linguist(dir, this.linguistJsOptions);
|
||||
const results = await this.getLinguistResults(dir);
|
||||
|
||||
try {
|
||||
const totalBytes = results.languages.bytes;
|
||||
@@ -233,9 +257,15 @@ export class LinguistBackendApi {
|
||||
await fs.remove(dir);
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
async getLinguistResults(dir: string): Promise<Results> {
|
||||
const results = await linguist(dir, this.linguistJsOptions);
|
||||
return results;
|
||||
}
|
||||
}
|
||||
|
||||
export function kindOrDefault(kind?: string[]) {
|
||||
export function kindOrDefault(kind?: string[]): string[] {
|
||||
if (!kind || kind.length === 0) {
|
||||
return ['API', 'Component', 'Template'];
|
||||
}
|
||||
@@ -14,4 +14,4 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { LinguistBackendApi } from './LinguistBackendApi';
|
||||
export type { LinguistBackendApi } from './LinguistBackendClient';
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Knex as KnexType, Knex } from 'knex';
|
||||
import { TestDatabases } from '@backstage/backend-test-utils';
|
||||
import {
|
||||
LinguistBackendDatabase,
|
||||
LinguistBackendStore,
|
||||
} from './LinguistBackendDatabase';
|
||||
import { Languages, ProcessedEntity } from '@backstage/plugin-linguist-common';
|
||||
|
||||
function createDatabaseManager(
|
||||
client: KnexType,
|
||||
skipMigrations: boolean = false,
|
||||
) {
|
||||
return {
|
||||
getClient: async () => client,
|
||||
migrations: {
|
||||
skip: skipMigrations,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const rawDbEntityResultRows = [
|
||||
{
|
||||
id: '14b32439-848a-49b2-92a4-98753f932606',
|
||||
entity_ref: 'template:default/create-react-app-template',
|
||||
languages: undefined,
|
||||
processed_date: undefined,
|
||||
},
|
||||
{
|
||||
id: '8c85d3ec-ccea-4b29-9de9-ccdd7e5ba452',
|
||||
entity_ref: 'template:default/docs-template',
|
||||
languages: undefined,
|
||||
processed_date: undefined,
|
||||
},
|
||||
{
|
||||
id: 'b922c87b-37bc-4505-b4af-70a1438decda',
|
||||
entity_ref: 'template:default/pull-request',
|
||||
languages:
|
||||
'{"languageCount":1,"totalBytes":2205,"processedDate":"2023-02-15T20:10:21.378Z","breakdown":[{"name":"YAML","percentage":100,"bytes":2205,"type":"data","color":"#cb171e"}]}',
|
||||
processed_date: new Date('2023-02-15 20:10:21.378Z'),
|
||||
},
|
||||
{
|
||||
id: 'bd555e6d-a3d0-4b48-a930-194db8f80db7',
|
||||
entity_ref: 'template:default/react-ssr-template',
|
||||
languages:
|
||||
'{"languageCount":8,"totalBytes":8988,"processedDate":"2023-02-15T20:10:21.388Z","breakdown":[{"name":"INI","percentage":2.26,"bytes":203,"type":"data","color":"#d1dbe0"},{"name":"JavaScript","percentage":5.94,"bytes":534,"type":"programming","color":"#f1e05a"},{"name":"YAML","percentage":31.09,"bytes":2794,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":11.79,"bytes":1060,"type":"prose","color":"#083fa1"},{"name":"JSON","percentage":21.09,"bytes":1896,"type":"data","color":"#292929"},{"name":"CSS","percentage":0,"bytes":0,"type":"markup","color":"#563d7c"},{"name":"TSX","percentage":26.01,"bytes":2338,"type":"programming","color":"#3178c6"},{"name":"TypeScript","percentage":1.81,"bytes":163,"type":"programming","color":"#3178c6"}]}',
|
||||
processed_date: new Date('2023-02-15 20:10:21.388Z'),
|
||||
},
|
||||
{
|
||||
id: '4145c0cf-44e9-4e95-9d57-781af4685b28',
|
||||
entity_ref: 'template:default/springboot-template',
|
||||
languages:
|
||||
'{"languageCount":9,"totalBytes":80986,"processedDate":"2023-02-15T20:10:21.419Z","breakdown":[{"name":"Shell","percentage":0.16,"bytes":130,"type":"programming","color":"#89e051"},{"name":"INI","percentage":0.35,"bytes":286,"type":"data","color":"#d1dbe0"},{"name":"Dockerfile","percentage":0.3,"bytes":246,"type":"programming","color":"#384d54"},{"name":"YAML","percentage":3.92,"bytes":3171,"type":"data","color":"#cb171e"},{"name":"Markdown","percentage":1.31,"bytes":1059,"type":"prose","color":"#083fa1"},{"name":"XML","percentage":10.48,"bytes":8491,"type":"data","color":"#0060ac"},{"name":"Java","percentage":1.8,"bytes":1455,"type":"programming","color":"#b07219"},{"name":"Text","percentage":81.22,"bytes":65780,"type":"prose"},{"name":"Protocol Buffer","percentage":0.45,"bytes":368,"type":"data"}]}',
|
||||
processed_date: new Date('2023-02-15 20:10:21.419Z'),
|
||||
},
|
||||
];
|
||||
|
||||
describe('Linguist database', () => {
|
||||
const databases = TestDatabases.create();
|
||||
let store: LinguistBackendStore;
|
||||
let testDbClient: Knex<any, unknown[]>;
|
||||
beforeAll(async () => {
|
||||
testDbClient = await databases.init('SQLITE_3');
|
||||
const database = createDatabaseManager(testDbClient);
|
||||
|
||||
store = await LinguistBackendDatabase.create(await database.getClient());
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await testDbClient.batchInsert('entity_result', rawDbEntityResultRows);
|
||||
});
|
||||
afterEach(async () => {
|
||||
await testDbClient('entity_result').delete();
|
||||
});
|
||||
|
||||
it('should be able to return entity results', async () => {
|
||||
const validLanguagesResult: Languages = {
|
||||
languageCount: 1,
|
||||
totalBytes: 2205,
|
||||
processedDate: '2023-02-15T20:10:21.378Z',
|
||||
breakdown: [
|
||||
{
|
||||
name: 'YAML',
|
||||
percentage: 100,
|
||||
bytes: 2205,
|
||||
type: 'data',
|
||||
color: '#cb171e',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const entityResult = await store.getEntityResults(
|
||||
'template:default/pull-request',
|
||||
);
|
||||
expect(entityResult).toMatchObject(validLanguagesResult);
|
||||
});
|
||||
|
||||
it('should return empty entity results when not found', async () => {
|
||||
const validEmptyLanguagesResult: Languages = {
|
||||
languageCount: 0,
|
||||
totalBytes: 0,
|
||||
processedDate: 'undefined',
|
||||
breakdown: [],
|
||||
};
|
||||
|
||||
const entityResult = await store.getEntityResults(
|
||||
'template:default/create-react-app-template',
|
||||
);
|
||||
expect(entityResult).toMatchObject(validEmptyLanguagesResult);
|
||||
});
|
||||
|
||||
it('should be able to return unprocessed entities', async () => {
|
||||
const validUnprocessedEntities: string[] = [
|
||||
'template:default/create-react-app-template',
|
||||
'template:default/docs-template',
|
||||
];
|
||||
|
||||
const unprocessedEntities = await store.getUnprocessedEntities();
|
||||
|
||||
expect(unprocessedEntities).toMatchObject(validUnprocessedEntities);
|
||||
});
|
||||
|
||||
it('should return string[] when there is no unprocessed entities', async () => {
|
||||
await testDbClient('entity_result').delete();
|
||||
const unprocessedEntities = await store.getUnprocessedEntities();
|
||||
|
||||
expect(unprocessedEntities).toMatchObject([]);
|
||||
});
|
||||
|
||||
it('should be able to return processed entities', async () => {
|
||||
const validProcessedEntities: ProcessedEntity[] = [
|
||||
{
|
||||
entityRef: 'template:default/pull-request',
|
||||
processedDate: new Date('2023-02-15 20:10:21.378Z'),
|
||||
},
|
||||
{
|
||||
entityRef: 'template:default/react-ssr-template',
|
||||
processedDate: new Date('2023-02-15 20:10:21.388Z'),
|
||||
},
|
||||
{
|
||||
entityRef: 'template:default/springboot-template',
|
||||
processedDate: new Date('2023-02-15 20:10:21.419Z'),
|
||||
},
|
||||
];
|
||||
|
||||
const processedEntities = await store.getProcessedEntities();
|
||||
|
||||
expect(processedEntities).toMatchObject(validProcessedEntities);
|
||||
});
|
||||
|
||||
it('should return string[] when there is no processed entities', async () => {
|
||||
await testDbClient('entity_result').delete();
|
||||
const unprocessedEntities = await store.getProcessedEntities();
|
||||
|
||||
expect(unprocessedEntities).toMatchObject([]);
|
||||
});
|
||||
|
||||
it('should insert new entities and ignore duplicates', async () => {
|
||||
const before = testDbClient.from('entity_result').count();
|
||||
|
||||
await store.insertNewEntity('component:/default/new-entity-one');
|
||||
await store.insertNewEntity('component:/default/new-entity-two');
|
||||
await store.insertNewEntity('template:default/pull-request');
|
||||
|
||||
const after = testDbClient.from('entity_result').count();
|
||||
|
||||
expect(before).toEqual(after);
|
||||
});
|
||||
|
||||
it('should get all entities', async () => {
|
||||
const allEntities = await store.getAllEntities();
|
||||
|
||||
expect(allEntities.length).toEqual(5);
|
||||
});
|
||||
|
||||
it('should delete entity by its entityRef', async () => {
|
||||
const before = await testDbClient.from('entity_result').count();
|
||||
|
||||
await store.deleteEntity('template:default/create-react-app-template');
|
||||
|
||||
const after = await testDbClient.from('entity_result').count();
|
||||
|
||||
expect(before).toEqual([{ 'count(*)': 5 }]);
|
||||
expect(after).toEqual([{ 'count(*)': 4 }]);
|
||||
});
|
||||
});
|
||||
@@ -26,8 +26,8 @@ import {
|
||||
export type RawDbEntityResultRow = {
|
||||
id: string;
|
||||
entity_ref: string;
|
||||
languages: string;
|
||||
processed_date: Date;
|
||||
languages?: string;
|
||||
processed_date?: Date;
|
||||
};
|
||||
|
||||
/** @public */
|
||||
@@ -35,8 +35,10 @@ export interface LinguistBackendStore {
|
||||
insertEntityResults(entityLanguages: EntityResults): Promise<string>;
|
||||
insertNewEntity(entityRef: string): Promise<void>;
|
||||
getEntityResults(entityRef: string): Promise<Languages>;
|
||||
getProcessedEntities(): Promise<ProcessedEntity[] | []>;
|
||||
getUnprocessedEntities(): Promise<string[] | []>;
|
||||
getProcessedEntities(): Promise<ProcessedEntity[]>;
|
||||
getUnprocessedEntities(): Promise<string[]>;
|
||||
getAllEntities(): Promise<string[]>;
|
||||
deleteEntity(entityRef: string): Promise<void>;
|
||||
}
|
||||
|
||||
const migrationsDir = resolvePackagePath(
|
||||
@@ -91,7 +93,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
.where({ entity_ref: entityRef })
|
||||
.first();
|
||||
|
||||
if (!entityResults) {
|
||||
if (!entityResults || !entityResults.languages) {
|
||||
const emptyResults: Languages = {
|
||||
languageCount: 0,
|
||||
totalBytes: 0,
|
||||
@@ -108,7 +110,7 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
}
|
||||
}
|
||||
|
||||
async getProcessedEntities(): Promise<ProcessedEntity[] | []> {
|
||||
async getProcessedEntities(): Promise<ProcessedEntity[]> {
|
||||
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result')
|
||||
.whereNotNull('processed_date')
|
||||
.whereNotNull('languages');
|
||||
@@ -118,9 +120,20 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
}
|
||||
|
||||
const processedEntities = rawEntities.map(rawEntity => {
|
||||
// Note: processed_date should never be null, this is handled by the DB query above
|
||||
let processedDate = new Date();
|
||||
if (rawEntity.processed_date) {
|
||||
// SQLite will return a Timestamp whereas Postgres will return a proper Date
|
||||
// This tests to see if we are getting a timestamp and convert if needed
|
||||
processedDate = new Date(+rawEntity.processed_date.toString());
|
||||
if (isNaN(+rawEntity.processed_date.toString())) {
|
||||
processedDate = rawEntity.processed_date;
|
||||
}
|
||||
}
|
||||
|
||||
const processEntity = {
|
||||
entityRef: rawEntity.entity_ref,
|
||||
processedDate: rawEntity.processed_date,
|
||||
processedDate,
|
||||
};
|
||||
|
||||
return processEntity;
|
||||
@@ -129,8 +142,11 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
return processedEntities;
|
||||
}
|
||||
|
||||
async getUnprocessedEntities(): Promise<string[] | []> {
|
||||
async getUnprocessedEntities(): Promise<string[]> {
|
||||
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result')
|
||||
// TODO(ahhhndre) processed_date should always be null as well but it had a default to the current date
|
||||
// once the default has been removed and released, we can then come back an enable this check
|
||||
// .whereNull('processed_date')
|
||||
.whereNull('languages')
|
||||
.orderBy('created_at', 'asc');
|
||||
|
||||
@@ -144,4 +160,24 @@ export class LinguistBackendDatabase implements LinguistBackendStore {
|
||||
|
||||
return unprocessedEntities;
|
||||
}
|
||||
|
||||
async getAllEntities(): Promise<string[]> {
|
||||
const rawEntities = await this.db<RawDbEntityResultRow>('entity_result');
|
||||
|
||||
if (!rawEntities) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const allEntities = rawEntities.map(rawEntity => {
|
||||
return rawEntity.entity_ref;
|
||||
});
|
||||
|
||||
return allEntities;
|
||||
}
|
||||
|
||||
async deleteEntity(entityRef: string): Promise<void> {
|
||||
await this.db<RawDbEntityResultRow>('entity_result')
|
||||
.where('entity_ref', entityRef)
|
||||
.delete();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,4 @@
|
||||
*/
|
||||
|
||||
export * from './service/router';
|
||||
export { LinguistBackendApi } from './api';
|
||||
export { LinguistBackendDatabase } from './db';
|
||||
export type { LinguistBackendStore } from './db';
|
||||
export type { LinguistBackendApi } from './api';
|
||||
|
||||
@@ -73,7 +73,7 @@ describe('createRouter', () => {
|
||||
const router = await createRouter(
|
||||
{ schedule: schedule, age: { days: 30 }, useSourceLocation: false },
|
||||
{
|
||||
linguistBackendApi,
|
||||
linguistBackendApi: linguistBackendApi,
|
||||
discovery: testDiscovery,
|
||||
database: createDatabase(),
|
||||
reader: mockUrlReader,
|
||||
|
||||
@@ -31,6 +31,8 @@ import {
|
||||
TaskScheduleDefinition,
|
||||
} from '@backstage/backend-tasks';
|
||||
import { HumanDuration } from '@backstage/types';
|
||||
import { CatalogClient } from '@backstage/catalog-client';
|
||||
import { LinguistBackendClient } from '../api/LinguistBackendClient';
|
||||
|
||||
/** @public */
|
||||
export interface PluginOptions {
|
||||
@@ -74,14 +76,16 @@ export async function createRouter(
|
||||
await database.getClient(),
|
||||
);
|
||||
|
||||
const linguistBackendApi =
|
||||
const catalogClient = new CatalogClient({ discoveryApi: discovery });
|
||||
|
||||
const linguistBackendClient =
|
||||
routerOptions.linguistBackendApi ||
|
||||
new LinguistBackendApi(
|
||||
new LinguistBackendClient(
|
||||
logger,
|
||||
linguistBackendStore,
|
||||
reader,
|
||||
discovery,
|
||||
tokenManager,
|
||||
catalogClient,
|
||||
age,
|
||||
batchSize,
|
||||
useSourceLocation,
|
||||
@@ -100,7 +104,7 @@ export async function createRouter(
|
||||
initialDelay: schedule.initialDelay,
|
||||
scope: schedule.scope,
|
||||
fn: async () => {
|
||||
await linguistBackendApi.processEntities();
|
||||
await linguistBackendClient.processEntities();
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -122,7 +126,7 @@ export async function createRouter(
|
||||
throw new Error('No entityRef was provided');
|
||||
}
|
||||
|
||||
const entityLanguages = await linguistBackendApi.getEntityLanguages(
|
||||
const entityLanguages = await linguistBackendClient.getEntityLanguages(
|
||||
entityRef as string,
|
||||
);
|
||||
res.status(200).json(entityLanguages);
|
||||
|
||||
@@ -169,67 +169,85 @@ export const useTaskEventStream = (taskId: string): TaskStream => {
|
||||
let didCancel = false;
|
||||
let subscription: Subscription | undefined;
|
||||
let logPusher: NodeJS.Timeout | undefined;
|
||||
|
||||
scaffolderApi.getTask(taskId).then(
|
||||
task => {
|
||||
if (didCancel) {
|
||||
return;
|
||||
}
|
||||
dispatch({ type: 'INIT', data: task });
|
||||
|
||||
// TODO(blam): Use a normal fetch to fetch the current log for the event stream
|
||||
// and use that for an INIT_EVENTs dispatch event, and then
|
||||
// use the last event ID to subscribe using after option to
|
||||
// stream logs. Without this, if you have a lot of logs, it can look like the
|
||||
// task is being rebuilt on load as it progresses through the steps at a slower
|
||||
// rate whilst it builds the status from the event logs
|
||||
const observable = scaffolderApi.streamLogs({ taskId });
|
||||
|
||||
const collectedLogEvents = new Array<LogEvent>();
|
||||
|
||||
function emitLogs() {
|
||||
if (collectedLogEvents.length) {
|
||||
const logs = collectedLogEvents.splice(
|
||||
0,
|
||||
collectedLogEvents.length,
|
||||
);
|
||||
dispatch({ type: 'LOGS', data: logs });
|
||||
let retryCount = 1;
|
||||
const startStreamLogProcess = () =>
|
||||
scaffolderApi.getTask(taskId).then(
|
||||
task => {
|
||||
if (didCancel) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
dispatch({ type: 'INIT', data: task });
|
||||
|
||||
logPusher = setInterval(emitLogs, 500);
|
||||
// TODO(blam): Use a normal fetch to fetch the current log for the event stream
|
||||
// and use that for an INIT_EVENTs dispatch event, and then
|
||||
// use the last event ID to subscribe using after option to
|
||||
// stream logs. Without this, if you have a lot of logs, it can look like the
|
||||
// task is being rebuilt on load as it progresses through the steps at a slower
|
||||
// rate whilst it builds the status from the event logs
|
||||
const observable = scaffolderApi.streamLogs({ taskId });
|
||||
|
||||
subscription = observable.subscribe({
|
||||
next: event => {
|
||||
switch (event.type) {
|
||||
case 'log':
|
||||
return collectedLogEvents.push(event);
|
||||
case 'cancelled':
|
||||
dispatch({ type: 'CANCELLED' });
|
||||
return undefined;
|
||||
case 'completion':
|
||||
emitLogs();
|
||||
dispatch({ type: 'COMPLETED', data: event });
|
||||
return undefined;
|
||||
default:
|
||||
throw new Error(
|
||||
`Unhandled event type ${event.type} in observer`,
|
||||
);
|
||||
const collectedLogEvents = new Array<LogEvent>();
|
||||
|
||||
function emitLogs() {
|
||||
if (collectedLogEvents.length) {
|
||||
const logs = collectedLogEvents.splice(
|
||||
0,
|
||||
collectedLogEvents.length,
|
||||
);
|
||||
dispatch({ type: 'LOGS', data: logs });
|
||||
}
|
||||
},
|
||||
error: error => {
|
||||
emitLogs();
|
||||
dispatch({ type: 'ERROR', data: error });
|
||||
},
|
||||
});
|
||||
},
|
||||
error => {
|
||||
if (!didCancel) {
|
||||
dispatch({ type: 'ERROR', data: error });
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
logPusher = setInterval(emitLogs, 500);
|
||||
|
||||
subscription = observable.subscribe({
|
||||
next: event => {
|
||||
switch (event.type) {
|
||||
case 'log':
|
||||
return collectedLogEvents.push(event);
|
||||
case 'cancelled':
|
||||
dispatch({ type: 'CANCELLED' });
|
||||
return undefined;
|
||||
case 'completion':
|
||||
emitLogs();
|
||||
dispatch({ type: 'COMPLETED', data: event });
|
||||
return undefined;
|
||||
default:
|
||||
throw new Error(
|
||||
`Unhandled event type ${event.type} in observer`,
|
||||
);
|
||||
}
|
||||
},
|
||||
error: error => {
|
||||
emitLogs();
|
||||
// in some cases the error is a refused connection from backend
|
||||
// this can happen from internet issues or proxy problems
|
||||
// so we try to reconnect again after some time
|
||||
// just to restart the fetch process
|
||||
// details here https://github.com/backstage/backstage/issues/15002
|
||||
|
||||
if (!error.message) {
|
||||
error.message = `We cannot connect at the moment, trying again in some seconds... Retrying (${retryCount}/3 retries)`;
|
||||
}
|
||||
|
||||
if (retryCount <= 3) {
|
||||
setTimeout(() => {
|
||||
retryCount += 1;
|
||||
startStreamLogProcess();
|
||||
}, 15000);
|
||||
}
|
||||
|
||||
dispatch({ type: 'ERROR', data: error });
|
||||
},
|
||||
});
|
||||
},
|
||||
error => {
|
||||
if (!didCancel) {
|
||||
dispatch({ type: 'ERROR', data: error });
|
||||
}
|
||||
},
|
||||
);
|
||||
startStreamLogProcess();
|
||||
return () => {
|
||||
didCancel = true;
|
||||
if (subscription) {
|
||||
|
||||
@@ -74,6 +74,7 @@
|
||||
"@types/react": "^16.13.1 || ^17.0.0",
|
||||
"@uiw/react-codemirror": "^4.9.3",
|
||||
"classnames": "^2.2.6",
|
||||
"event-source-polyfill": "^1.0.31",
|
||||
"git-url-parse": "^13.0.0",
|
||||
"humanize-duration": "^3.25.1",
|
||||
"immer": "^9.0.1",
|
||||
@@ -108,7 +109,6 @@
|
||||
"@types/json-schema": "^7.0.9",
|
||||
"@types/node": "^16.11.26",
|
||||
"cross-fetch": "^3.1.5",
|
||||
"event-source-polyfill": "1.0.25",
|
||||
"msw": "^1.0.0"
|
||||
},
|
||||
"files": [
|
||||
|
||||
@@ -20,11 +20,14 @@ import { MockFetchApi, setupRequestMockHandlers } from '@backstage/test-utils';
|
||||
import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { ScaffolderClient } from './api';
|
||||
import { EventSourcePolyfill } from 'event-source-polyfill';
|
||||
|
||||
const MockedEventSource = global.EventSource as jest.MockedClass<
|
||||
typeof EventSource
|
||||
const MockedEventSource = EventSourcePolyfill as jest.MockedClass<
|
||||
typeof EventSourcePolyfill
|
||||
>;
|
||||
|
||||
jest.mock('event-source-polyfill');
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
describe('api', () => {
|
||||
@@ -102,6 +105,9 @@ describe('api', () => {
|
||||
},
|
||||
);
|
||||
|
||||
const token = 'fake-token';
|
||||
identityApi.getCredentials.mockResolvedValue({ token: token });
|
||||
|
||||
const next = jest.fn();
|
||||
|
||||
await new Promise<void>(complete => {
|
||||
@@ -112,7 +118,10 @@ describe('api', () => {
|
||||
|
||||
expect(MockedEventSource).toHaveBeenCalledWith(
|
||||
'http://backstage/api/v2/tasks/a-random-task-id/eventstream',
|
||||
{ withCredentials: true },
|
||||
{
|
||||
withCredentials: true,
|
||||
headers: { Authorization: `Bearer ${token}` },
|
||||
},
|
||||
);
|
||||
expect(MockedEventSource.prototype.close).toHaveBeenCalled();
|
||||
|
||||
|
||||
@@ -41,6 +41,7 @@ import {
|
||||
} from '@backstage/plugin-scaffolder-react';
|
||||
|
||||
import queryString from 'qs';
|
||||
import { EventSourcePolyfill } from 'event-source-polyfill';
|
||||
|
||||
/**
|
||||
* An API to interact with the scaffolder backend.
|
||||
@@ -224,8 +225,11 @@ export class ScaffolderClient implements ScaffolderApi {
|
||||
params.set('after', String(Number(after)));
|
||||
}
|
||||
|
||||
this.discoveryApi.getBaseUrl('scaffolder').then(
|
||||
baseUrl => {
|
||||
Promise.all([
|
||||
this.discoveryApi.getBaseUrl('scaffolder'),
|
||||
this.identityApi?.getCredentials(),
|
||||
]).then(
|
||||
([baseUrl, credentials]) => {
|
||||
const url = `${baseUrl}/v2/tasks/${encodeURIComponent(
|
||||
taskId,
|
||||
)}/eventstream`;
|
||||
@@ -240,7 +244,12 @@ export class ScaffolderClient implements ScaffolderApi {
|
||||
}
|
||||
};
|
||||
|
||||
const eventSource = new EventSource(url, { withCredentials: true });
|
||||
const eventSource = new EventSourcePolyfill(url, {
|
||||
withCredentials: true,
|
||||
headers: credentials?.token
|
||||
? { Authorization: `Bearer ${credentials.token}` }
|
||||
: {},
|
||||
});
|
||||
eventSource.addEventListener('log', processEvent);
|
||||
eventSource.addEventListener('cancelled', processEvent);
|
||||
eventSource.addEventListener('completion', (event: any) => {
|
||||
|
||||
@@ -20,11 +20,14 @@ import { ReactNode } from 'react';
|
||||
import { ResultHighlight } from '@backstage/plugin-search-common';
|
||||
import { RouteRef } from '@backstage/core-plugin-api';
|
||||
import { SearchResultListItemExtensionProps } from '@backstage/plugin-search-react';
|
||||
import { SyncResult as SyncResult_2 } from '@backstage/plugin-techdocs-react';
|
||||
import { TableColumn } from '@backstage/core-components';
|
||||
import { TableOptions } from '@backstage/core-components';
|
||||
import { TableProps } from '@backstage/core-components';
|
||||
import { TechDocsApi as TechDocsApi_2 } from '@backstage/plugin-techdocs-react';
|
||||
import { TechDocsEntityMetadata as TechDocsEntityMetadata_2 } from '@backstage/plugin-techdocs-react';
|
||||
import { TechDocsMetadata as TechDocsMetadata_2 } from '@backstage/plugin-techdocs-react';
|
||||
import { TechDocsStorageApi as TechDocsStorageApi_2 } from '@backstage/plugin-techdocs-react';
|
||||
import { ToolbarProps } from '@material-ui/core';
|
||||
import { UserListFilterKind } from '@backstage/plugin-catalog-react';
|
||||
|
||||
@@ -237,7 +240,7 @@ export interface TechDocsApi {
|
||||
export const techdocsApiRef: ApiRef<TechDocsApi>;
|
||||
|
||||
// @public
|
||||
export class TechDocsClient implements TechDocsApi {
|
||||
export class TechDocsClient implements TechDocsApi_2 {
|
||||
constructor(options: {
|
||||
configApi: Config;
|
||||
discoveryApi: DiscoveryApi;
|
||||
@@ -442,7 +445,7 @@ export interface TechDocsStorageApi {
|
||||
export const techdocsStorageApiRef: ApiRef<TechDocsStorageApi>;
|
||||
|
||||
// @public
|
||||
export class TechDocsStorageClient implements TechDocsStorageApi {
|
||||
export class TechDocsStorageClient implements TechDocsStorageApi_2 {
|
||||
constructor(options: {
|
||||
configApi: Config;
|
||||
discoveryApi: DiscoveryApi;
|
||||
@@ -471,6 +474,6 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
|
||||
syncEntityDocs(
|
||||
entityId: CompoundEntityRef,
|
||||
logHandler?: (line: string) => void,
|
||||
): Promise<SyncResult>;
|
||||
): Promise<SyncResult_2>;
|
||||
}
|
||||
```
|
||||
|
||||
@@ -23,11 +23,13 @@ import {
|
||||
} from '@backstage/core-plugin-api';
|
||||
import { NotFoundError, ResponseError } from '@backstage/errors';
|
||||
import {
|
||||
SyncResult,
|
||||
TechDocsApi,
|
||||
TechDocsEntityMetadata,
|
||||
TechDocsMetadata,
|
||||
TechDocsStorageApi,
|
||||
} from '@backstage/plugin-techdocs-react';
|
||||
import { EventSourcePolyfill } from 'event-source-polyfill';
|
||||
import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api';
|
||||
|
||||
/**
|
||||
* API to talk to `techdocs-backend`.
|
||||
@@ -189,7 +191,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
|
||||
* @param entityId - Object containing entity data like name, namespace, etc.
|
||||
* @param logHandler - Callback to receive log messages from the build process
|
||||
* @returns Whether documents are currently synchronized to newest version
|
||||
* @throws Throws error on error from sync endpoint in Techdocs Backend
|
||||
* @throws Throws error on error from sync endpoint in TechDocs Backend
|
||||
*/
|
||||
async syncEntityDocs(
|
||||
entityId: CompoundEntityRef,
|
||||
|
||||
@@ -29,7 +29,7 @@ export type TechDocsPageWrapperProps = {
|
||||
};
|
||||
|
||||
/**
|
||||
* Component wrapping a techdocs page with Page and Header components
|
||||
* Component wrapping a TechDocs page with Page and Header components
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
The TechDocs reader is a component that fetches a remote page, runs transformers on it and renders it into a shadow dom.
|
||||
|
||||
Currently there's no easy way to customize which transformers to run or add new ones. If that is needed you would have to fork the techdocs plugin and make your changes in that fork.
|
||||
Currently there's no easy way to customize which transformers to run or add new ones. If that is needed you would have to fork the TechDocs plugin and make your changes in that fork.
|
||||
|
||||
Transformers are functions that optionally takes in parameters from the Reader.tsx component and returns a function which gets passed the DOM of the fetched page. A very simple transformer can look like this.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user