paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Rewrite the /sync/:namespace/:kind/:name to return an event-stream

Browse Source 
Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
This commit is contained in:
Dominik Henneke
2021-07-05 12:43:23 +02:00
parent c18e8eb910
commit f1200f44c8
15 changed files with 691 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/techdocs/api-report.md
+3 -3
View File
@@ -44,7 +44,7 @@ export const Reader: ({ entityId, onReady }: Props_2) => JSX.Element;
export const Router: () => JSX.Element;
// @public (undocumented)
export type SyncResult = 'cached' | 'updated' | 'timeout';
export type SyncResult = 'cached' | 'updated';
// @public (undocumented)
export interface TechDocsApi {
@@ -112,7 +112,7 @@ export interface TechDocsStorageApi {
// (undocumented)
getStorageUrl(): Promise<string>;
// (undocumented)
syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise<SyncResult>;
}
// @public (undocumented)
@@ -140,7 +140,7 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
getStorageUrl(): Promise<string>;
// (undocumented)
identityApi: IdentityApi;
syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
syncEntityDocs(entityId: EntityName, logHandler?: (line: string) => void): Promise<SyncResult>;
}
plugins/techdocs/dev/index.tsx
+12 -9
View File
@@ -17,6 +17,7 @@
import { createDevApp } from '@backstage/dev-utils';
import { NotFoundError } from '@backstage/errors';
import React from 'react';
import { EntityName } from '@backstage/catalog-model';
import {
Reader,
SyncResult,
@@ -81,9 +82,18 @@ function createPage({
});
}
async syncEntityDocs() {
async syncEntityDocs(_: EntityName, logHandler?: (line: string) => void) {
if (syncDocsDelay) {
await new Promise(resolve => setTimeout(resolve, syncDocsDelay));
for (let i = 0; i < 10; i++) {
setTimeout(
() => logHandler?.call(this, `Log line ${i}`),
((i + 1) * syncDocsDelay) / 10,
);
}
await new Promise(resolve => {
setTimeout(resolve, syncDocsDelay);
});
}
return syncDocs();
@@ -195,13 +205,6 @@ createDevApp()
syncDocsDelay: 2000,
})}
</TabbedLayout.Route>
<TabbedLayout.Route path="/timeout" title="Sync Timeout">
{createPage({
syncDocs: () => 'timeout',
syncDocsDelay: 2000,
})}
</TabbedLayout.Route>
</TabbedLayout>
</Page>
),
plugins/techdocs/package.json
+2
View File
@@ -44,6 +44,7 @@
"@material-ui/icons": "^4.9.1",
"@material-ui/lab": "4.0.0-alpha.45",
"@material-ui/styles": "^4.10.0",
"eventsource": "^1.1.0",
"react": "^16.13.1",
"react-dom": "^16.13.1",
"react-router": "6.0.0-beta.0",
@@ -60,6 +61,7 @@
"@testing-library/react": "^11.2.5",
"@testing-library/react-hooks": "^3.4.2",
"@testing-library/user-event": "^13.1.8",
"@types/eventsource": "^1.1.5",
"@types/jest": "^26.0.7",
"@types/node": "^14.14.32",
"@types/react": "^16.9",
plugins/techdocs/src/api.ts
+5 -2
View File
@@ -28,14 +28,17 @@ export const techdocsApiRef = createApiRef<TechDocsApi>({
description: 'Used to make requests towards techdocs API',
});
export type SyncResult = 'cached' | 'updated' | 'timeout';
export type SyncResult = 'cached' | 'updated';
export interface TechDocsStorageApi {
getApiOrigin(): Promise<string>;
getStorageUrl(): Promise<string>;
getBuilder(): Promise<string>;
getEntityDocs(entityId: EntityName, path: string): Promise<string>;
syncEntityDocs(entityId: EntityName): Promise<SyncResult>;
syncEntityDocs(
entityId: EntityName,
logHandler?: (line: string) => void,
): Promise<SyncResult>;
getBaseUrl(
oldBaseUrl: string,
entityId: EntityName,
plugins/techdocs/src/client.test.ts
+200 -1
View File
@@ -13,9 +13,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Config } from '@backstage/config';
import { TechDocsStorageClient } from './client';
import { UrlPatternDiscovery } from '@backstage/core-app-api';
import { IdentityApi } from '@backstage/core-plugin-api';
import { NotFoundError } from '@backstage/errors';
import EventSource from 'eventsource';
import { TechDocsStorageClient } from './client';
const MockedEventSource: jest.MockedClass<
typeof EventSource
> = EventSource as any;
jest.mock('eventsource');
const mockEntity = {
kind: 'Component',
@@ -29,6 +39,16 @@ describe('TechDocsStorageClient', () => {
getOptionalString: () => 'http://backstage:9191/api/techdocs',
} as Partial<Config>;
const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
const identityApi: jest.Mocked<IdentityApi> = {
getIdToken: jest.fn(),
getProfile: jest.fn(),
getUserId: jest.fn(),
signOut: jest.fn(),
};
beforeEach(() => {
jest.resetAllMocks();
});
it('should return correct base url based on defined storage', async () => {
// @ts-ignore Partial<Config> not assignable to Config.
@@ -51,4 +71,183 @@ describe('TechDocsStorageClient', () => {
`${mockBaseUrl}/static/docs/${mockEntity.namespace}/${mockEntity.kind}/${mockEntity.name}/test/`,
);
});
describe('syncEntityDocs', () => {
it('should create eventsource without headers', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
MockedEventSource.prototype.addEventListener.mockImplementation(
(type, fn) => {
if (type === 'finish') {
fn({ data: '{"updated": false}' } as any);
}
},
);
await storageApi.syncEntityDocs(mockEntity);
expect(
MockedEventSource,
).toBeCalledWith(
'http://backstage:9191/api/techdocs/sync/default/Component/test-component',
{ withCredentials: true, headers: {} },
);
});
it('should create eventsource with headers', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
MockedEventSource.prototype.addEventListener.mockImplementation(
(type, fn) => {
if (type === 'finish') {
fn({ data: '{"updated": false}' } as any);
}
},
);
identityApi.getIdToken.mockResolvedValue('token');
await storageApi.syncEntityDocs(mockEntity);
expect(
MockedEventSource,
).toBeCalledWith(
'http://backstage:9191/api/techdocs/sync/default/Component/test-component',
{ withCredentials: true, headers: { Authorization: 'Bearer token' } },
);
});
it('should resolve to cached', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
MockedEventSource.prototype.addEventListener.mockImplementation(
(type, fn) => {
if (type === 'finish') {
fn({ data: '{"updated": false}' } as any);
}
},
);
await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual(
'cached',
);
});
it('should resolve to updated', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
MockedEventSource.prototype.addEventListener.mockImplementation(
(type, fn) => {
if (type === 'finish') {
fn({ data: '{"updated": true}' } as any);
}
},
);
await expect(storageApi.syncEntityDocs(mockEntity)).resolves.toEqual(
'updated',
);
});
it('should log values', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
MockedEventSource.prototype.addEventListener.mockImplementation(
(type, fn) => {
if (type === 'log') {
fn({ data: '"A log message"' } as any);
}
if (type === 'finish') {
fn({ data: '{"updated": false}' } as any);
}
},
);
const logHandler = jest.fn();
await expect(
storageApi.syncEntityDocs(mockEntity, logHandler),
).resolves.toEqual('cached');
expect(logHandler).toBeCalledTimes(1);
expect(logHandler).toBeCalledWith('A log message');
});
it('should throw NotFoundError', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
// we await later after we emitted the error
const promise = storageApi.syncEntityDocs(mockEntity).then();
// flush the event loop
await new Promise(setImmediate);
const instance = MockedEventSource.mock
.instances[0] as jest.Mocked<EventSource>;
instance.onerror({
status: 404,
message: 'Some not found warning',
} as any);
await expect(promise).rejects.toThrow(NotFoundError);
await expect(promise).rejects.toThrowError('Some not found warning');
});
it('should throw generic errors', async () => {
const storageApi = new TechDocsStorageClient({
// @ts-ignore Partial<Config> not assignable to Config.
configApi,
discoveryApi,
identityApi,
});
// we await later after we emitted the error
const promise = storageApi.syncEntityDocs(mockEntity).then();
// flush the event loop
await new Promise(setImmediate);
const instance = MockedEventSource.mock
.instances[0] as jest.Mocked<EventSource>;
instance.onerror({
status: 500,
message: 'Some other error',
} as any);
await expect(promise).rejects.toThrow(Error);
await expect(promise).rejects.toThrowError('Some other error');
});
});
});
plugins/techdocs/src/client.ts
+41 -25
View File
@@ -16,10 +16,11 @@
import { EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api';
import { NotFoundError } from '@backstage/errors';
import EventSource from 'eventsource';
import { SyncResult, TechDocsApi, TechDocsStorageApi } from './api';
import { TechDocsEntityMetadata, TechDocsMetadata } from './types';
import { DiscoveryApi, IdentityApi } from '@backstage/core-plugin-api';
/**
* API to talk to techdocs-backend.
@@ -192,44 +193,59 @@ export class TechDocsStorageClient implements TechDocsStorageApi {
* Check if docs are on the latest version and trigger rebuild if not
*
* @param {EntityName} entityId Object containing entity data like name, namespace, etc.
* @param {Function} logHandler Callback to receive log messages from the build process
* @returns {SyncResult} Whether documents are currently synchronized to newest version
* @throws {Error} Throws error on error from sync endpoint in Techdocs Backend
*/
async syncEntityDocs(entityId: EntityName): Promise<SyncResult> {
async syncEntityDocs(
entityId: EntityName,
logHandler: (line: string) => void = () => {},
): Promise<SyncResult> {
const { kind, namespace, name } = entityId;
const apiOrigin = await this.getApiOrigin();
const url = `${apiOrigin}/sync/${namespace}/${kind}/${name}`;
const token = await this.identityApi.getIdToken();
let request;
let attempts: number = 0;
// retry if request times out, up to 5 times
// can happen due to docs taking too long to generate
while (!request || (request.status === 408 && attempts < 5)) {
attempts++;
request = await fetch(url, {
return new Promise((resolve, reject) => {
const source = new EventSource(url, {
withCredentials: true,
headers: token ? { Authorization: `Bearer ${token}` } : {},
});
}
switch (request.status) {
case 404:
throw new NotFoundError((await request.json()).error);
source.addEventListener('log', (e: any) => {
if (e.data) {
logHandler(JSON.parse(e.data));
}
});
case 200:
case 304:
return 'cached';
source.addEventListener('finish', (e: any) => {
let updated: boolean = false;
case 201:
return 'updated';
if (e.data) {
({ updated } = JSON.parse(e.data));
}
// for timeout and misc errors, handle without error to allow viewing older docs
// if older docs not available,
// Reader will show 404 error coming from getEntityDocs
case 408:
default:
return 'timeout';
}
resolve(updated ? 'updated' : 'cached');
});
source.onerror = (e: any) => {
source.close();
switch (e.status) {
// the endpoint returned a 404 status
case 404:
reject(new NotFoundError(e.message));
return;
// also handles the event-stream close. the reject is ignored if the Promise was already
// resolved by a finish event.
default:
reject(new Error(e.message));
return;
}
};
});
}
async getBaseUrl(
plugins/techdocs/src/reader/components/Reader.tsx
+1 -7
View File
@@ -15,6 +15,7 @@
*/
import { EntityName } from '@backstage/catalog-model';
import { useApi } from '@backstage/core-plugin-api';
import { scmIntegrationsApiRef } from '@backstage/integration-react';
import { BackstageTheme } from '@backstage/theme';
import { useTheme } from '@material-ui/core';
@@ -37,7 +38,6 @@ import {
import { TechDocsNotFound } from './TechDocsNotFound';
import TechDocsProgressBar from './TechDocsProgressBar';
import { useReaderState } from './useReaderState';
import { useApi } from '@backstage/core-plugin-api';
type Props = {
entityId: EntityName;
@@ -328,12 +328,6 @@ export const Reader = ({ entityId, onReady }: Props) => {
to view.
</Alert>
)}
{state === 'CONTENT_STALE_TIMEOUT' && (
<Alert variant="outlined" severity="warning">
Building a newer version of this documentation took longer than
expected. Please refresh to try again.
</Alert>
)}
{state === 'CONTENT_STALE_ERROR' && (
<Alert variant="outlined" severity="error">
Building a newer version of this documentation failed. {errorMessage}
plugins/techdocs/src/reader/components/useReaderState.test.tsx
+1 -39
View File
@@ -14,6 +14,7 @@
* limitations under the License.
*/
import { ApiProvider, ApiRegistry } from '@backstage/core-app-api';
import { NotFoundError } from '@backstage/errors';
import { act, renderHook } from '@testing-library/react-hooks';
import React from 'react';
@@ -23,7 +24,6 @@ import {
reducer,
useReaderState,
} from './useReaderState';
import { ApiProvider, ApiRegistry } from '@backstage/core-app-api';
describe('useReaderState', () => {
let Wrapper: React.ComponentType;
@@ -55,14 +55,12 @@ describe('useReaderState', () => {
${false} | ${undefined} | ${'BUILDING'} | ${'INITIAL_BUILD'}
${false} | ${undefined} | ${'BUILD_READY'} | ${'CONTENT_NOT_FOUND'}
${false} | ${undefined} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'}
${false} | ${undefined} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_NOT_FOUND'}
${false} | ${undefined} | ${'UP_TO_DATE'} | ${'CONTENT_NOT_FOUND'}
${false} | ${undefined} | ${'ERROR'} | ${'CONTENT_NOT_FOUND'}
${false} | ${'asdf'} | ${'CHECKING'} | ${'CONTENT_FRESH'}
${false} | ${'asdf'} | ${'BUILDING'} | ${'CONTENT_STALE_REFRESHING'}
${false} | ${'asdf'} | ${'BUILD_READY'} | ${'CONTENT_STALE_READY'}
${false} | ${'asdf'} | ${'BUILD_READY_RELOAD'} | ${'CHECKING'}
${false} | ${'asdf'} | ${'BUILD_TIMED_OUT'} | ${'CONTENT_STALE_TIMEOUT'}
${false} | ${'asdf'} | ${'UP_TO_DATE'} | ${'CONTENT_FRESH'}
${false} | ${'asdf'} | ${'ERROR'} | ${'CONTENT_STALE_ERROR'}
`(
@@ -369,42 +367,6 @@ describe('useReaderState', () => {
});
});
it('should handle timed-out refresh', async () => {
techdocsStorageApi.getEntityDocs.mockResolvedValue('my content');
techdocsStorageApi.syncEntityDocs.mockResolvedValue('timeout');
await act(async () => {
const { result, waitForValueToChange } = await renderHook(
() => useReaderState('Component', 'default', 'backstage', '/example'),
{ wrapper: Wrapper },
);
expect(result.current).toEqual({
state: 'CHECKING',
content: undefined,
errorMessage: '',
});
// the content is returned but the sync is in progress
await waitForValueToChange(() => result.current.state);
expect(result.current).toEqual({
state: 'CONTENT_STALE_TIMEOUT',
content: 'my content',
errorMessage: '',
});
expect(techdocsStorageApi.getEntityDocs).toBeCalledWith(
{ kind: 'Component', namespace: 'default', name: 'backstage' },
'/example',
);
expect(techdocsStorageApi.syncEntityDocs).toBeCalledWith({
kind: 'Component',
namespace: 'default',
name: 'backstage',
});
});
});
it('should handle content error', async () => {
techdocsStorageApi.getEntityDocs.mockRejectedValue(
new NotFoundError('Some error description'),
plugins/techdocs/src/reader/components/useReaderState.ts
+22 -24
View File
@@ -14,10 +14,10 @@
* limitations under the License.
*/
import { useApi } from '@backstage/core-plugin-api';
import { useEffect, useMemo, useReducer, useRef } from 'react';
import { useAsync, useAsyncRetry } from 'react-use';
import { techdocsStorageApiRef } from '../../api';
import { useApi } from '@backstage/core-plugin-api';
/**
* A state representation that is used to configure the UI of <Reader />
@@ -35,9 +35,6 @@ type ContentStateTypes =
/** There is content, but after a reload, the content will be different */
| 'CONTENT_STALE_READY'
/** There is content, the backend tried to update it, but it took too long */
| 'CONTENT_STALE_TIMEOUT'
/** There is content, the backend tried to update it, but failed */
| 'CONTENT_STALE_ERROR'
@@ -93,11 +90,6 @@ export function calculateDisplayState({
return 'CONTENT_STALE_READY';
}
// the build timed out, but the content is still stale
if (activeSyncState === 'BUILD_TIMED_OUT') {
return 'CONTENT_STALE_TIMEOUT';
}
// the build failed, but the content is still stale
if (activeSyncState === 'ERROR') {
return 'CONTENT_STALE_ERROR';
@@ -127,9 +119,6 @@ type SyncStates =
*/
| 'BUILD_READY_RELOAD'
/** Building the documentation timed out */
| 'BUILD_TIMED_OUT'
/** No need for a sync. The content was already up-to-date. */
| 'UP_TO_DATE'
@@ -274,18 +263,27 @@ export function useReaderState(
name,
});
if (result === 'updated') {
// if there was no content prior to building, retry the loading
if (!contentRef.current.content) {
contentRef.current.reload();
dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' });
} else {
dispatch({ type: 'sync', state: 'BUILD_READY' });
}
} else if (result === 'cached') {
dispatch({ type: 'sync', state: 'UP_TO_DATE' });
} else {
dispatch({ type: 'sync', state: 'BUILD_TIMED_OUT' });
switch (result) {
case 'updated':
// if there was no content prior to building, retry the loading
if (!contentRef.current.content) {
contentRef.current.reload();
dispatch({ type: 'sync', state: 'BUILD_READY_RELOAD' });
} else {
dispatch({ type: 'sync', state: 'BUILD_READY' });
}
break;
case 'cached':
dispatch({ type: 'sync', state: 'UP_TO_DATE' });
break;
default:
dispatch({
type: 'sync',
state: 'ERROR',
syncError: new Error('Unexpected return state'),
});
break;
}
} catch (e) {
dispatch({ type: 'sync', state: 'ERROR', syncError: e });