Merge pull request #5450 from backstage/emmaindal/search-index-refresh-schedule

[Search] Extend search backend with scheduler

plugins/search-backend-node/src/IndexBuilder.test.ts

@@ -54,29 +54,33 @@ describe('IndexBuilder', () => {
 
   describe('addCollator', () => {
     it('adds a collator', async () => {
+      jest.useFakeTimers();
       const collatorSpy = jest.spyOn(testCollator, 'execute');
 
       // Add a collator.
       testIndexBuilder.addCollator({
         type: 'anything',
-        defaultRefreshIntervalSeconds: 600,
+        defaultRefreshIntervalSeconds: 6,
         collator: testCollator,
       });
 
       // Build the index and ensure the collator was invoked.
-      await testIndexBuilder.build();
+      const { scheduler } = await testIndexBuilder.build();
+      scheduler.start();
+      jest.advanceTimersByTime(6000);
       expect(collatorSpy).toHaveBeenCalled();
     });
   });
 
   describe('addDecorator', () => {
     it('adds a decorator', async () => {
+      jest.useFakeTimers();
       const decoratorSpy = jest.spyOn(testDecorator, 'execute');
 
       // Add a collator.
       testIndexBuilder.addCollator({
         type: 'anything',
-        defaultRefreshIntervalSeconds: 600,
+        defaultRefreshIntervalSeconds: 6,
         collator: testCollator,
       });
 
@@ -86,11 +90,16 @@ describe('IndexBuilder', () => {
       });
 
       // Build the index and ensure the decorator was invoked.
-      await testIndexBuilder.build();
+      const { scheduler } = await testIndexBuilder.build();
+      scheduler.start();
+      jest.advanceTimersByTime(6000);
+      // wait for async decorator execution
+      await Promise.resolve();
       expect(decoratorSpy).toHaveBeenCalled();
     });
 
     it('adds a type-specific decorator', async () => {
+      jest.useFakeTimers();
       const expectedType = 'an-expected-type';
       const docFixture = {
         title: 'Test',
@@ -105,7 +114,7 @@ describe('IndexBuilder', () => {
       // Add a collator.
       testIndexBuilder.addCollator({
         type: expectedType,
-        defaultRefreshIntervalSeconds: 600,
+        defaultRefreshIntervalSeconds: 6,
         collator: testCollator,
       });
 
@@ -116,7 +125,11 @@ describe('IndexBuilder', () => {
       });
 
       // Build the index and ensure the decorator was invoked.
-      await testIndexBuilder.build();
+      const { scheduler } = await testIndexBuilder.build();
+      scheduler.start();
+      jest.advanceTimersByTime(6000);
+      // wait for async decorator execution
+      await Promise.resolve();
       expect(decoratorSpy).toHaveBeenCalled();
       expect(decoratorSpy).toHaveBeenCalledWith([docFixture]);
     });
@@ -128,7 +141,7 @@ describe('IndexBuilder', () => {
         text: 'Test text.',
         location: '/test/location',
       };
-      jest
+      const collatorSpy = jest
         .spyOn(testCollator, 'execute')
         .mockImplementation(async () => [docFixture]);
       const decoratorSpy = jest.spyOn(testDecorator, 'execute');
@@ -136,7 +149,7 @@ describe('IndexBuilder', () => {
       // Add a collator.
       testIndexBuilder.addCollator({
         type: expectedType,
-        defaultRefreshIntervalSeconds: 600,
+        defaultRefreshIntervalSeconds: 6,
         collator: testCollator,
       });
 
@@ -147,7 +160,10 @@ describe('IndexBuilder', () => {
       });
 
       // Build the index and ensure the decorator was not invoked.
-      await testIndexBuilder.build();
+      const { scheduler } = await testIndexBuilder.build();
+      scheduler.start();
+      jest.advanceTimersByTime(6000);
+      expect(collatorSpy).toHaveBeenCalled();
       expect(decoratorSpy).not.toHaveBeenCalled();
     });
   });

plugins/search-backend-node/src/IndexBuilder.ts

@@ -16,7 +16,7 @@
 
 import { DocumentCollator, DocumentDecorator } from '@backstage/search-common';
 import { Logger } from 'winston';
-
+import { Scheduler } from './index';
 import {
   RegisterCollatorParameters,
   RegisterDecoratorParameters,
@@ -92,37 +92,42 @@ export class IndexBuilder {
   }
 
   /**
-   * Starts the process of executing collators and decorators and building the
-   * search index.
-   *
-   * TODO: But like with coordination, timing, error handling, and what have you.
+   * Compiles collators and decorators into tasks, which are added to a
+   * scheduler returned to the caller.
    */
-  async build() {
-    return Promise.all(
-      Object.keys(this.collators).map(async type => {
+  async build(): Promise<{ scheduler: Scheduler }> {
+    const scheduler = new Scheduler({ logger: this.logger });
+
+    Object.keys(this.collators).forEach(type => {
+      scheduler.addToSchedule(async () => {
+        // Collate, Decorate, Index.
         const decorators: DocumentDecorator[] = (
           this.decorators['*'] || []
         ).concat(this.decorators[type] || []);
 
-        this.logger.info(
+        this.logger.debug(
           `Collating documents for ${type} via ${this.collators[type].collate.constructor.name}`,
         );
         let documents = await this.collators[type].collate.execute();
         for (let i = 0; i < decorators.length; i++) {
-          this.logger.info(
+          this.logger.debug(
             `Decorating ${type} documents via ${decorators[i].constructor.name}`,
           );
           documents = await decorators[i].execute(documents);
         }
 
         if (!documents || documents.length === 0) {
-          this.logger.info(`No documents for type "${type}" to index`);
+          this.logger.debug(`No documents for type "${type}" to index`);
           return;
         }
 
         // pushing documents to index to a configured search engine.
         this.searchEngine.index(type, documents);
-      }),
-    );
+      }, this.collators[type].refreshInterval * 1000);
+    });
+
+    return {
+      scheduler,
+    };
   }
 }

plugins/search-backend-node/src/Scheduler.test.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { Scheduler } from './index';
+
+describe('Scheduler', () => {
+  let testScheduler: Scheduler;
+
+  beforeEach(() => {
+    const logger = getVoidLogger();
+    testScheduler = new Scheduler({
+      logger,
+    });
+  });
+
+  describe('addToSchedule', () => {
+    it('should not add a task and interval to schedule, if already started', async () => {
+      jest.useFakeTimers();
+      const mockTask1 = jest.fn();
+      const mockTask2 = jest.fn();
+
+      // Add a task and interval to schedule
+      testScheduler.addToSchedule(mockTask1, 2);
+
+      // Starts scheduling process
+      testScheduler.start();
+
+      // Throws Error if task and interval is added to a already started schedule
+      expect(() => testScheduler.addToSchedule(mockTask2, 2)).toThrowError();
+
+      jest.runOnlyPendingTimers();
+      expect(mockTask1).toHaveBeenCalled();
+      expect(mockTask2).not.toHaveBeenCalled();
+    });
+
+    it('should be possible to add a task and interval to schedule, if already started, but stopped in between', async () => {
+      jest.useFakeTimers();
+      const mockTask1 = jest.fn();
+      const mockTask2 = jest.fn();
+
+      // Add a task and interval to schedule
+      testScheduler.addToSchedule(mockTask1, 2);
+
+      // Starts scheduling process
+      testScheduler.start();
+
+      // Stop scheduling process
+      testScheduler.stop();
+
+      // Should't throw error, as it is stopped.
+      expect(() =>
+        testScheduler.addToSchedule(mockTask2, 4),
+      ).not.toThrowError();
+
+      // Starts scheduling process
+      testScheduler.start();
+
+      jest.runOnlyPendingTimers();
+      expect(mockTask1).toHaveBeenCalled();
+      expect(mockTask2).toHaveBeenCalled();
+    });
+  });
+});

plugins/search-backend-node/src/Scheduler.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Logger } from 'winston';
+
+type TaskEnvelope = {
+  task: Function;
+  interval: number;
+};
+
+/**
+ * TODO: coordination, error handling
+ */
+
+export class Scheduler {
+  private logger: Logger;
+  private schedule: TaskEnvelope[];
+  private intervalTimeouts: NodeJS.Timeout[] = [];
+
+  constructor({ logger }: { logger: Logger }) {
+    this.logger = logger;
+    this.schedule = [];
+  }
+
+  /**
+   * Adds each task and interval to the schedule
+   *
+   */
+  addToSchedule(task: Function, interval: number) {
+    if (this.intervalTimeouts.length) {
+      throw new Error(
+        'Cannot add task to schedule that has already been started.',
+      );
+    }
+    this.schedule.push({ task, interval });
+  }
+
+  /**
+   * Starts the scheduling process for each task
+   */
+  start() {
+    this.logger.info('Starting all scheduled search tasks.');
+    this.schedule.forEach(({ task, interval }) => {
+      this.intervalTimeouts.push(
+        setInterval(() => {
+          task();
+        }, interval),
+      );
+    });
+  }
+
+  /**
+   * Stop all scheduled tasks.
+   */
+  stop() {
+    this.logger.info('Stopping all scheduled search tasks.');
+    this.intervalTimeouts.forEach(timeout => {
+      clearInterval(timeout);
+    });
+    this.intervalTimeouts = [];
+  }
+}

plugins/search-backend-node/src/index.ts

@@ -15,5 +15,6 @@
  */
 
 export { IndexBuilder } from './IndexBuilder';
+export { Scheduler } from './Scheduler';
 export { LunrSearchEngine } from './engines';
 export type { SearchEngine } from './types';