diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 8ec575d6ec..9c29e45ca3 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -27,6 +27,7 @@ import { EntityPolicyProcessor } from './processors/EntityPolicyProcessor'; import { FileReaderProcessor } from './processors/FileReaderProcessor'; import { GithubReaderProcessor } from './processors/GithubReaderProcessor'; import { GithubApiReaderProcessor } from './processors/GithubApiReaderProcessor'; +import { GitlabApiReaderProcessor } from './processors/GitlabApiReaderProcessor'; import { GitlabReaderProcessor } from './processors/GitlabReaderProcessor'; import { LocationRefProcessor } from './processors/LocationEntityProcessor'; import * as result from './processors/results'; @@ -59,6 +60,7 @@ export class LocationReaders implements LocationReader { new FileReaderProcessor(), new GithubReaderProcessor(), new GithubApiReaderProcessor(), + new GitlabApiReaderProcessor(), new GitlabReaderProcessor(), new YamlProcessor(), new EntityPolicyProcessor(entityPolicy), @@ -102,7 +104,6 @@ export class LocationReaders implements LocationReader { }); } } - if (newItems.length === 0) { return output; } diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts new file mode 100644 index 0000000000..00abaab430 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, LocationSpec } from '@backstage/catalog-model'; +import fetch, { RequestInit, HeadersInit } from 'node-fetch'; +import lodash from 'lodash'; +import yaml from 'yaml'; +import * as result from './results'; +import { LocationProcessor, LocationProcessorEmit } from './types'; + +export class GitlabApiReaderProcessor implements LocationProcessor { + private privateToken: string = process.env.GITLAB_PRIVATE_TOKEN || ''; + + getRequestOptions(): RequestInit { + const headers: HeadersInit = { 'PRIVATE-TOKEN': '' }; + if (this.privateToken !== '') { + headers['PRIVATE-TOKEN'] = this.privateToken; + } + + const requestOptions: RequestInit = { + headers, + }; + + return requestOptions; + } + + async readLocation( + location: LocationSpec, + optional: boolean, + emit: LocationProcessorEmit, + ): Promise { + if (location.type !== 'gitlab/api') { + return false; + } + + try { + // const url = this.buildRawUrl(location.target); + const url = new URL(location.target); + const response = await fetch(url.toString(), this.getRequestOptions()); + if (response.ok) { + const data = await response.buffer(); + emit(result.data(location, data)); + } else { + const message = `${location.target} could not be read as ${url}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + if (!optional) { + emit(result.notFoundError(location, message)); + } + } else { + emit(result.generalError(location, message)); + } + } + } catch (e) { + const message = `Unable to read ${location.type} ${location.target}, ${e}`; + emit(result.generalError(location, message)); + } + return true; + } + + // We need our own parseData because the gitlab api url has `/raw?ref=branch` + // on the end which doesn't match the regex in YamlProcessor (.ya?ml$) + async parseData( + data: Buffer, + location: LocationSpec, + emit: LocationProcessorEmit, + ): Promise { + if (!location.target.match(/\.ya?ml/)) { + return false; + } + + let documents: yaml.Document.Parsed[]; + try { + documents = yaml.parseAllDocuments(data.toString('utf8')).filter(d => d); + } catch (e) { + emit(result.generalError(location, `Failed to parse YAML, ${e}`)); + return true; + } + + for (const document of documents) { + if (document.errors?.length) { + const message = `YAML error, ${document.errors[0]}`; + emit(result.generalError(location, message)); + } else { + const json = document.toJSON(); + if (lodash.isPlainObject(json)) { + emit(result.entity(location, json as Entity)); + } else { + const message = `Expected object at root, got ${typeof json}`; + emit(result.generalError(location, message)); + } + } + } + + return true; + } +}