diff --git a/.changeset/search-odd-ads-invite.md b/.changeset/search-odd-ads-invite.md new file mode 100644 index 0000000000..0dde3d0c1f --- /dev/null +++ b/.changeset/search-odd-ads-invite.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-node': patch +--- + +Introduced Scheduler which is responsible for adding new tasks to a schedule together with it's interval timer as well as starting and stopping the scheduler processes. diff --git a/packages/backend/src/plugins/search.ts b/packages/backend/src/plugins/search.ts index a980661c85..dcd48aaf40 100644 --- a/packages/backend/src/plugins/search.ts +++ b/packages/backend/src/plugins/search.ts @@ -35,11 +35,10 @@ export default async function createPlugin({ collator: new DefaultCatalogCollator(discovery), }); - // TODO: Move refresh loop logic into the builder. - const timerId = setInterval(() => { - indexBuilder.build(); - }, 60000); - useHotCleanup(module, () => clearInterval(timerId)); + const { scheduler } = await indexBuilder.build(); + + scheduler.start(); + useHotCleanup(module, () => scheduler.stop()); return await createRouter({ engine: indexBuilder.getSearchEngine(), diff --git a/plugins/search-backend-node/src/index.test.ts b/plugins/search-backend-node/src/IndexBuilder.test.ts similarity index 81% rename from plugins/search-backend-node/src/index.test.ts rename to plugins/search-backend-node/src/IndexBuilder.test.ts index 0c66db372a..e1dc511b97 100644 --- a/plugins/search-backend-node/src/index.test.ts +++ b/plugins/search-backend-node/src/IndexBuilder.test.ts @@ -54,29 +54,33 @@ describe('IndexBuilder', () => { describe('addCollator', () => { it('adds a collator', async () => { + jest.useFakeTimers(); const collatorSpy = jest.spyOn(testCollator, 'execute'); // Add a collator. testIndexBuilder.addCollator({ type: 'anything', - defaultRefreshIntervalSeconds: 600, + defaultRefreshIntervalSeconds: 6, collator: testCollator, }); // Build the index and ensure the collator was invoked. - await testIndexBuilder.build(); + const { scheduler } = await testIndexBuilder.build(); + scheduler.start(); + jest.advanceTimersByTime(6000); expect(collatorSpy).toHaveBeenCalled(); }); }); describe('addDecorator', () => { it('adds a decorator', async () => { + jest.useFakeTimers(); const decoratorSpy = jest.spyOn(testDecorator, 'execute'); // Add a collator. testIndexBuilder.addCollator({ type: 'anything', - defaultRefreshIntervalSeconds: 600, + defaultRefreshIntervalSeconds: 6, collator: testCollator, }); @@ -86,11 +90,16 @@ describe('IndexBuilder', () => { }); // Build the index and ensure the decorator was invoked. - await testIndexBuilder.build(); + const { scheduler } = await testIndexBuilder.build(); + scheduler.start(); + jest.advanceTimersByTime(6000); + // wait for async decorator execution + await Promise.resolve(); expect(decoratorSpy).toHaveBeenCalled(); }); it('adds a type-specific decorator', async () => { + jest.useFakeTimers(); const expectedType = 'an-expected-type'; const docFixture = { title: 'Test', @@ -105,7 +114,7 @@ describe('IndexBuilder', () => { // Add a collator. testIndexBuilder.addCollator({ type: expectedType, - defaultRefreshIntervalSeconds: 600, + defaultRefreshIntervalSeconds: 6, collator: testCollator, }); @@ -116,7 +125,11 @@ describe('IndexBuilder', () => { }); // Build the index and ensure the decorator was invoked. - await testIndexBuilder.build(); + const { scheduler } = await testIndexBuilder.build(); + scheduler.start(); + jest.advanceTimersByTime(6000); + // wait for async decorator execution + await Promise.resolve(); expect(decoratorSpy).toHaveBeenCalled(); expect(decoratorSpy).toHaveBeenCalledWith([docFixture]); }); @@ -128,7 +141,7 @@ describe('IndexBuilder', () => { text: 'Test text.', location: '/test/location', }; - jest + const collatorSpy = jest .spyOn(testCollator, 'execute') .mockImplementation(async () => [docFixture]); const decoratorSpy = jest.spyOn(testDecorator, 'execute'); @@ -136,7 +149,7 @@ describe('IndexBuilder', () => { // Add a collator. testIndexBuilder.addCollator({ type: expectedType, - defaultRefreshIntervalSeconds: 600, + defaultRefreshIntervalSeconds: 6, collator: testCollator, }); @@ -147,7 +160,10 @@ describe('IndexBuilder', () => { }); // Build the index and ensure the decorator was not invoked. - await testIndexBuilder.build(); + const { scheduler } = await testIndexBuilder.build(); + scheduler.start(); + jest.advanceTimersByTime(6000); + expect(collatorSpy).toHaveBeenCalled(); expect(decoratorSpy).not.toHaveBeenCalled(); }); }); diff --git a/plugins/search-backend-node/src/IndexBuilder.ts b/plugins/search-backend-node/src/IndexBuilder.ts index 0f5a287037..71d374e704 100644 --- a/plugins/search-backend-node/src/IndexBuilder.ts +++ b/plugins/search-backend-node/src/IndexBuilder.ts @@ -16,7 +16,7 @@ import { DocumentCollator, DocumentDecorator } from '@backstage/search-common'; import { Logger } from 'winston'; - +import { Scheduler } from './index'; import { RegisterCollatorParameters, RegisterDecoratorParameters, @@ -92,37 +92,42 @@ export class IndexBuilder { } /** - * Starts the process of executing collators and decorators and building the - * search index. - * - * TODO: But like with coordination, timing, error handling, and what have you. + * Compiles collators and decorators into tasks, which are added to a + * scheduler returned to the caller. */ - async build() { - return Promise.all( - Object.keys(this.collators).map(async type => { + async build(): Promise<{ scheduler: Scheduler }> { + const scheduler = new Scheduler({ logger: this.logger }); + + Object.keys(this.collators).forEach(type => { + scheduler.addToSchedule(async () => { + // Collate, Decorate, Index. const decorators: DocumentDecorator[] = ( this.decorators['*'] || [] ).concat(this.decorators[type] || []); - this.logger.info( + this.logger.debug( `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`, ); let documents = await this.collators[type].collate.execute(); for (let i = 0; i < decorators.length; i++) { - this.logger.info( + this.logger.debug( `Decorating ${type} documents via ${decorators[i].constructor.name}`, ); documents = await decorators[i].execute(documents); } if (!documents || documents.length === 0) { - this.logger.info(`No documents for type "${type}" to index`); + this.logger.debug(`No documents for type "${type}" to index`); return; } // pushing documents to index to a configured search engine. this.searchEngine.index(type, documents); - }), - ); + }, this.collators[type].refreshInterval * 1000); + }); + + return { + scheduler, + }; } } diff --git a/plugins/search-backend-node/src/Scheduler.test.ts b/plugins/search-backend-node/src/Scheduler.test.ts new file mode 100644 index 0000000000..43fdb7ea92 --- /dev/null +++ b/plugins/search-backend-node/src/Scheduler.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { Scheduler } from './index'; + +describe('Scheduler', () => { + let testScheduler: Scheduler; + + beforeEach(() => { + const logger = getVoidLogger(); + testScheduler = new Scheduler({ + logger, + }); + }); + + describe('addToSchedule', () => { + it('should not add a task and interval to schedule, if already started', async () => { + jest.useFakeTimers(); + const mockTask1 = jest.fn(); + const mockTask2 = jest.fn(); + + // Add a task and interval to schedule + testScheduler.addToSchedule(mockTask1, 2); + + // Starts scheduling process + testScheduler.start(); + + // Throws Error if task and interval is added to a already started schedule + expect(() => testScheduler.addToSchedule(mockTask2, 2)).toThrowError(); + + jest.runOnlyPendingTimers(); + expect(mockTask1).toHaveBeenCalled(); + expect(mockTask2).not.toHaveBeenCalled(); + }); + + it('should be possible to add a task and interval to schedule, if already started, but stopped in between', async () => { + jest.useFakeTimers(); + const mockTask1 = jest.fn(); + const mockTask2 = jest.fn(); + + // Add a task and interval to schedule + testScheduler.addToSchedule(mockTask1, 2); + + // Starts scheduling process + testScheduler.start(); + + // Stop scheduling process + testScheduler.stop(); + + // Should't throw error, as it is stopped. + expect(() => + testScheduler.addToSchedule(mockTask2, 4), + ).not.toThrowError(); + + // Starts scheduling process + testScheduler.start(); + + jest.runOnlyPendingTimers(); + expect(mockTask1).toHaveBeenCalled(); + expect(mockTask2).toHaveBeenCalled(); + }); + }); +}); diff --git a/plugins/search-backend-node/src/Scheduler.ts b/plugins/search-backend-node/src/Scheduler.ts new file mode 100644 index 0000000000..e54a837cee --- /dev/null +++ b/plugins/search-backend-node/src/Scheduler.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Logger } from 'winston'; + +type TaskEnvelope = { + task: Function; + interval: number; +}; + +/** + * TODO: coordination, error handling + */ + +export class Scheduler { + private logger: Logger; + private schedule: TaskEnvelope[]; + private intervalTimeouts: NodeJS.Timeout[] = []; + + constructor({ logger }: { logger: Logger }) { + this.logger = logger; + this.schedule = []; + } + + /** + * Adds each task and interval to the schedule + * + */ + addToSchedule(task: Function, interval: number) { + if (this.intervalTimeouts.length) { + throw new Error( + 'Cannot add task to schedule that has already been started.', + ); + } + this.schedule.push({ task, interval }); + } + + /** + * Starts the scheduling process for each task + */ + start() { + this.logger.info('Starting all scheduled search tasks.'); + this.schedule.forEach(({ task, interval }) => { + this.intervalTimeouts.push( + setInterval(() => { + task(); + }, interval), + ); + }); + } + + /** + * Stop all scheduled tasks. + */ + stop() { + this.logger.info('Stopping all scheduled search tasks.'); + this.intervalTimeouts.forEach(timeout => { + clearInterval(timeout); + }); + this.intervalTimeouts = []; + } +} diff --git a/plugins/search-backend-node/src/index.ts b/plugins/search-backend-node/src/index.ts index a3a0eb855e..7ee75e3d59 100644 --- a/plugins/search-backend-node/src/index.ts +++ b/plugins/search-backend-node/src/index.ts @@ -15,5 +15,6 @@ */ export { IndexBuilder } from './IndexBuilder'; +export { Scheduler } from './Scheduler'; export { LunrSearchEngine } from './engines'; export type { SearchEngine } from './types';