filter folders from app-config file

Signed-off-by: kim5566 <28945404+kim5566@users.noreply.github.com>

plugins/todo-backend/src/lib/TodoReader/TodoScmReader.test.ts

@@ -189,4 +189,65 @@ describe('TodoScmReader', () => {
       'Failed to parse TODO in https://github.com/o/r/catalog-info.yaml at my-file.sh, Error: failed to parse',
     );
   });
+
+  it('should filter out exclude folders', async () => {
+    const reader = mockReader();
+    const excludeFolders = ['vendor/'];
+    const todoReader = new TodoScmReader({
+      logger: getVoidLogger(),
+      reader,
+      integrations: ScmIntegrations.fromConfig(new ConfigReader({})),
+      excludeFolders: excludeFolders,
+    });
+    reader.readTree.mockResolvedValueOnce({
+      files: async () => [
+        {
+          content: async () => Buffer.from('// TODO: my-todo', 'utf8'),
+          path: 'vendor/another-file.go',
+        },
+        {
+          content: async () => Buffer.from('// TODO: my-todo', 'utf8'),
+          path: 'my-folder/my-file.js',
+        },
+      ],
+    } as ReadTreeResponse);
+    await expect(
+      todoReader.readTodos({
+        url: 'https://github.com/backstage/backstage/catalog-info.yaml',
+      }),
+    ).resolves.toEqual({
+      items: [
+        {
+          text: 'my-todo',
+          tag: 'TODO',
+          lineNumber: 1,
+          repoFilePath: 'vendor/another-file.go',
+          viewUrl:
+            'https://github.com/backstage/backstage/vendor/another-file.go#L1',
+        },
+        {
+          text: 'my-todo',
+          tag: 'TODO',
+          lineNumber: 1,
+          repoFilePath: 'my-folder/my-file.js',
+          viewUrl:
+            'https://github.com/backstage/backstage/my-folder/my-file.js#L1',
+        },
+      ],
+    });
+    expect(reader.readTree).toHaveBeenCalledTimes(1);
+    expect(reader.readTree).toHaveBeenCalledWith(
+      'https://github.com/backstage/backstage/catalog-info.yaml',
+      {
+        etag: undefined,
+        filter: expect.any(Function),
+      },
+    );
+    // Filter function should filter out exclude folders
+    const filterFunc = reader.readTree.mock.calls[0][1]!.filter!;
+    expect(filterFunc('another-file.go')).toBe(true);
+    expect(filterFunc('vendor/another-file.go')).toBe(false);
+    expect(filterFunc('my-file.js')).toBe(true);
+    expect(filterFunc('my-folder/my-file.js')).toBe(true);
+  });
 });

plugins/todo-backend/src/lib/TodoReader/TodoScmReader.ts

@@ -47,6 +47,7 @@ export type TodoScmReaderOptions = {
   reader: UrlReader;
   integrations: ScmIntegrations;
   parser?: TodoParser;
+  excludeFolders?: string[];
 };
 
 type CacheItem = {
@@ -60,6 +61,7 @@ export class TodoScmReader implements TodoReader {
   private readonly reader: UrlReader;
   private readonly parser: TodoParser;
   private readonly integrations: ScmIntegrations;
+  private readonly excludeFolders: string[];
 
   private readonly cache = new Map<string, CacheItem>();
   private readonly inFlightReads = new Map<string, Promise<CacheItem>>();
@@ -68,8 +70,11 @@ export class TodoScmReader implements TodoReader {
     config: Config,
     options: Omit<TodoScmReaderOptions, 'integrations'>,
   ) {
+    const excludeFolders: string[] =
+      config.getOptionalStringArray('todo.excludeFolders') ?? [];
     return new TodoScmReader({
       ...options,
+      excludeFolders,
       integrations: ScmIntegrations.fromConfig(config),
     });
   }
@@ -79,6 +84,7 @@ export class TodoScmReader implements TodoReader {
     this.reader = options.reader;
     this.parser = options.parser ?? createTodoParser();
     this.integrations = options.integrations;
+    this.excludeFolders = options.excludeFolders ?? [];
   }
 
   async readTodos(options: ReadTodosOptions): Promise<ReadTodosResult> {
@@ -89,7 +95,12 @@ export class TodoScmReader implements TodoReader {
     }
 
     const cacheItem = this.cache.get(url);
-    const newRead = this.doReadTodos({ url }, cacheItem?.etag).catch(error => {
+    const excludeFolders = this.excludeFolders;
+    const newRead = this.doReadTodos(
+      { url },
+      excludeFolders,
+      cacheItem?.etag,
+    ).catch(error => {
       if (cacheItem && error.name === 'NotModifiedError') {
         return cacheItem;
       }
@@ -108,6 +119,7 @@ export class TodoScmReader implements TodoReader {
 
   private async doReadTodos(
     options: ReadTodosOptions,
+    excludeFolders?: string[],
     etag?: string,
   ): Promise<CacheItem> {
     const { url } = options;
@@ -118,10 +130,12 @@ export class TodoScmReader implements TodoReader {
         if (info && info.size > MAX_FILE_SIZE) {
           return false;
         }
+        const excFolders = excludeFolders ?? [];
         return (
           !filePath.startsWith('.') &&
           !filePath.includes('/.') &&
-          !excludedExtensions.includes(extname)
+          !excludedExtensions.includes(extname) &&
+          !excFolders.some(exclude => filePath.startsWith(exclude))
         );
       },
     });

yarn.lock

@@ -15740,9 +15740,9 @@ kleur@^4.0.3:
   integrity sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA==
 
 knex@^1.0.2:
-  version "1.0.2"
-  resolved "https://registry.npmjs.org/knex/-/knex-1.0.2.tgz#1b79273f39f587a631c1a5515482c203d5971781"
-  integrity sha512-RuDKTylj6X/3nYomnsFV8sOdxTcehLHczOd3yrUdULE4pQR8jVlZxYt3vvIU04otJF0Cw9DCtRt05S4PN4kDpw==
+  version "1.0.3"
+  resolved "https://registry.npmjs.org/knex/-/knex-1.0.3.tgz#a5f97aa98e5e036cfd0209a90d53b2a411280e84"
+  integrity sha512-rY1T7cgTQGHAUD9TshMka37bd+SEK+koPXXvZQEIoE8yjJ/E8ShsenaAmr3oaNNzqXuKD/SC0qlYtp7Js8tAXA==
   dependencies:
     colorette "2.0.16"
     commander "^8.3.0"