diff --git a/.changeset/many-bags-sort.md b/.changeset/many-bags-sort.md new file mode 100644 index 0000000000..dbc8df8b0c --- /dev/null +++ b/.changeset/many-bags-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Support globs in `FileReaderProcessor`. diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index 254898a086..c595c8dd5a 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -45,6 +45,7 @@ "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", "git-url-parse": "^11.4.4", + "glob": "^7.1.6", "knex": "^0.21.6", "ldapjs": "^2.2.0", "lodash": "^4.17.15", diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts new file mode 100644 index 0000000000..8ea165dcb8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FileReaderProcessor } from './FileReaderProcessor'; +import { + CatalogProcessorEntityResult, + CatalogProcessorErrorResult, + CatalogProcessorResult, +} from './types'; + +describe('FileReaderProcessor', () => { + const fixturesRoot = + 'src/ingestion/processors/__fixtures__/fileReaderProcessor'; + + it('should load from file', async () => { + const processor = new FileReaderProcessor(); + const spec = { + type: 'file', + target: `${fixturesRoot}/component.yaml`, + }; + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as CatalogProcessorEntityResult; + + expect(generated.type).toBe('entity'); + expect(generated.location).toEqual(spec); + expect(generated.entity).toEqual({ kind: 'Component' }); + }); + + it('should fail load from file with error', async () => { + const processor = new FileReaderProcessor(); + const spec = { + type: 'file', + target: `${fixturesRoot}/missing.yaml`, + }; + + const generated = (await new Promise(emit => + processor.readLocation(spec, false, emit), + )) as CatalogProcessorErrorResult; + + expect(generated.type).toBe('error'); + expect(generated.location).toBe(spec); + expect(generated.error.name).toBe('NotFoundError'); + expect(generated.error.message).toBe( + `file ${fixturesRoot}/missing.yaml does not exist`, + ); + }); + + it('should support globs', async () => { + const processor = new FileReaderProcessor(); + + const emit = jest.fn(); + + await processor.readLocation( + { type: 'file', target: `${fixturesRoot}/**/*.yaml` }, + false, + emit, + ); + + expect(emit).toBeCalledTimes(2); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts index 2328ec3c7a..db069913a2 100644 --- a/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/FileReaderProcessor.ts @@ -19,6 +19,10 @@ import fs from 'fs-extra'; import * as result from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; import { parseEntityYaml } from './util/parse'; +import { promisify } from 'util'; +import g from 'glob'; + +const glob = promisify(g); export class FileReaderProcessor implements CatalogProcessor { async readLocation( @@ -31,12 +35,15 @@ export class FileReaderProcessor implements CatalogProcessor { } try { - const exists = await fs.pathExists(location.target); - if (exists) { - const data = await fs.readFile(location.target); + const fileMatches = await glob(location.target); - for (const parseResult of parseEntityYaml(data, location)) { - emit(parseResult); + if (fileMatches.length > 0) { + for (const fileMatch of fileMatches) { + const data = await fs.readFile(fileMatch); + + for (const parseResult of parseEntityYaml(data, location)) { + emit(parseResult); + } } } else if (!optional) { const message = `${location.type} ${location.target} does not exist`; diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml new file mode 100644 index 0000000000..8524aaf14a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/component.yaml @@ -0,0 +1 @@ +kind: Component diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml new file mode 100644 index 0000000000..a894e34f70 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/dir/api.yaml @@ -0,0 +1 @@ +kind: API diff --git a/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt new file mode 100644 index 0000000000..f332206601 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/__fixtures__/fileReaderProcessor/test.txt @@ -0,0 +1 @@ +Just a text file