Merge pull request #4534 from SDA-SE/feat/file-reader-glob

Support globs in FileReaderProcessor

.changeset/many-bags-sort.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Support globs in `FileReaderProcessor`.

plugins/catalog-backend/package.json

@@ -45,6 +45,7 @@
     "express-promise-router": "^3.0.3",
     "fs-extra": "^9.0.0",
     "git-url-parse": "^11.4.4",
+    "glob": "^7.1.6",
     "knex": "^0.21.6",
     "ldapjs": "^2.2.0",
     "lodash": "^4.17.15",

plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { FileReaderProcessor } from './FileReaderProcessor';
+import {
+  CatalogProcessorEntityResult,
+  CatalogProcessorErrorResult,
+  CatalogProcessorResult,
+} from './types';
+import path from 'path';
+
+describe('FileReaderProcessor', () => {
+  const fixturesRoot = path.join(
+    'src',
+    'ingestion',
+    'processors',
+    '__fixtures__',
+    'fileReaderProcessor',
+  );
+
+  it('should load from file', async () => {
+    const processor = new FileReaderProcessor();
+    const spec = {
+      type: 'file',
+      target: `${path.join(fixturesRoot, 'component.yaml')}`,
+    };
+
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit),
+    )) as CatalogProcessorEntityResult;
+
+    expect(generated.type).toBe('entity');
+    expect(generated.location).toEqual(spec);
+    expect(generated.entity).toEqual({ kind: 'Component' });
+  });
+
+  it('should fail load from file with error', async () => {
+    const processor = new FileReaderProcessor();
+    const spec = {
+      type: 'file',
+      target: `${path.join(fixturesRoot, 'missing.yaml')}`,
+    };
+
+    const generated = (await new Promise<CatalogProcessorResult>(emit =>
+      processor.readLocation(spec, false, emit),
+    )) as CatalogProcessorErrorResult;
+
+    expect(generated.type).toBe('error');
+    expect(generated.location).toBe(spec);
+    expect(generated.error.name).toBe('NotFoundError');
+    expect(generated.error.message).toBe(
+      `file ${path.join(fixturesRoot, 'missing.yaml')} does not exist`,
+    );
+  });
+
+  it('should support globs', async () => {
+    const processor = new FileReaderProcessor();
+
+    const emit = jest.fn();
+
+    await processor.readLocation(
+      { type: 'file', target: `${path.join(fixturesRoot, '**', '*.yaml')}` },
+      false,
+      emit,
+    );
+
+    expect(emit).toBeCalledTimes(2);
+    expect(emit.mock.calls[0][0].entity).toEqual({ kind: 'Component' });
+    expect(emit.mock.calls[1][0].entity).toEqual({ kind: 'API' });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts

@@ -19,6 +19,10 @@ import fs from 'fs-extra';
 import * as result from './results';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 import { parseEntityYaml } from './util/parse';
+import { promisify } from 'util';
+import g from 'glob';
+
+const glob = promisify(g);
 
 export class FileReaderProcessor implements CatalogProcessor {
   async readLocation(
@@ -31,12 +35,15 @@ export class FileReaderProcessor implements CatalogProcessor {
     }
 
     try {
-      const exists = await fs.pathExists(location.target);
-      if (exists) {
-        const data = await fs.readFile(location.target);
+      const fileMatches = await glob(location.target);
 
-        for (const parseResult of parseEntityYaml(data, location)) {
-          emit(parseResult);
+      if (fileMatches.length > 0) {
+        for (const fileMatch of fileMatches) {
+          const data = await fs.readFile(fileMatch);
+
+          for (const parseResult of parseEntityYaml(data, location)) {
+            emit(parseResult);
+          }
         }
       } else if (!optional) {
         const message = `${location.type} ${location.target} does not exist`;

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml

@@ -0,0 +1 @@
+kind: Component

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml

@@ -0,0 +1 @@
+kind: API

plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt

@@ -0,0 +1 @@
+Just a text file