From a0787d59a48d67c5cdf8a457ee6996e0f59d7553 Mon Sep 17 00:00:00 2001 From: Forrest Waters Date: Wed, 22 Jul 2020 16:13:42 -0500 Subject: [PATCH] lookup gitlab project ID with gitlab API and add tests --- .../src/ingestion/LocationReaders.ts | 1 + .../GitlabApiReaderProcessor.test.ts | 94 ++++++++++++++++++ .../processors/GitlabApiReaderProcessor.ts | 96 ++++++++++++------- 3 files changed, 155 insertions(+), 36 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts diff --git a/plugins/catalog-backend/src/ingestion/LocationReaders.ts b/plugins/catalog-backend/src/ingestion/LocationReaders.ts index 9c29e45ca3..545465cde0 100644 --- a/plugins/catalog-backend/src/ingestion/LocationReaders.ts +++ b/plugins/catalog-backend/src/ingestion/LocationReaders.ts @@ -104,6 +104,7 @@ export class LocationReaders implements LocationReader { }); } } + if (newItems.length === 0) { return output; } diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts new file mode 100644 index 0000000000..a429a0e6fb --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { GitlabApiReaderProcessor } from './GitlabApiReaderProcessor'; + +describe('GitlabApiReaderProcessor', () => { + it('should build raw api', () => { + const processor = new GitlabApiReaderProcessor(); + + const tests = [ + { + target: + 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: new URL( + 'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml', + url: new URL( + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup + url: new URL( + 'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml?ref=branch', + ), + err: undefined, + }, + { + target: + 'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/', + url: null, + err: + 'Incorrect url: https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/, Error: Gitlab url does not end in .ya?ml', + }, + ]; + + for (const test of tests) { + if (test.err) { + expect(() => processor.buildRawUrl(test.target, 12345)).toThrowError( + test.err, + ); + } else { + expect(processor.buildRawUrl(test.target, 12345)).toEqual(test.url); + } + } + }); + + it('should return request options', () => { + const tests = [ + { + token: '0123456789', + expect: { + headers: { + 'PRIVATE-TOKEN': '0123456789', + }, + }, + }, + { + token: '', + expect: { + headers: { + 'PRIVATE-TOKEN': '', + }, + }, + }, + ]; + + for (const test of tests) { + process.env.GITLAB_PRIVATE_TOKEN = test.token; + const processor = new GitlabApiReaderProcessor(); + expect(processor.getRequestOptions()).toEqual(test.expect); + } + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts index 00abaab430..6964640eb4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitlabApiReaderProcessor.ts @@ -14,10 +14,8 @@ * limitations under the License. */ -import { Entity, LocationSpec } from '@backstage/catalog-model'; +import { LocationSpec } from '@backstage/catalog-model'; import fetch, { RequestInit, HeadersInit } from 'node-fetch'; -import lodash from 'lodash'; -import yaml from 'yaml'; import * as result from './results'; import { LocationProcessor, LocationProcessorEmit } from './types'; @@ -47,8 +45,8 @@ export class GitlabApiReaderProcessor implements LocationProcessor { } try { - // const url = this.buildRawUrl(location.target); - const url = new URL(location.target); + const projectID = await this.getProjectID(location.target); + const url = this.buildRawUrl(location.target, projectID); const response = await fetch(url.toString(), this.getRequestOptions()); if (response.ok) { const data = await response.buffer(); @@ -70,40 +68,66 @@ export class GitlabApiReaderProcessor implements LocationProcessor { return true; } - // We need our own parseData because the gitlab api url has `/raw?ref=branch` - // on the end which doesn't match the regex in YamlProcessor (.ya?ml$) - async parseData( - data: Buffer, - location: LocationSpec, - emit: LocationProcessorEmit, - ): Promise { - if (!location.target.match(/\.ya?ml/)) { - return false; - } - - let documents: yaml.Document.Parsed[]; + // convert https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // to https://gitlab.com/api/v4/projects//repository/files/filepath?ref=branch + buildRawUrl(target: string, projectID: Number): URL { try { - documents = yaml.parseAllDocuments(data.toString('utf8')).filter(d => d); - } catch (e) { - emit(result.generalError(location, `Failed to parse YAML, ${e}`)); - return true; - } + const url = new URL(target); - for (const document of documents) { - if (document.errors?.length) { - const message = `YAML error, ${document.errors[0]}`; - emit(result.generalError(location, message)); - } else { - const json = document.toJSON(); - if (lodash.isPlainObject(json)) { - emit(result.entity(location, json as Entity)); - } else { - const message = `Expected object at root, got ${typeof json}`; - emit(result.generalError(location, message)); - } + const branchAndfilePath = url.pathname.split('/-/blob/')[1]; + + if (!branchAndfilePath.match(/\.ya?ml$/)) { + throw new Error('Gitlab url does not end in .ya?ml'); } - } - return true; + const [branch, ...filePath] = branchAndfilePath.split('/'); + + url.pathname = [ + '/api/v4/projects', + projectID, + 'repository/files', + encodeURIComponent(filePath.join('/')), + 'raw', + ].join('/'); + url.search = `?ref=${branch}`; + + return url; + } catch (e) { + throw new Error(`Incorrect url: ${target}, ${e}`); + } + } + + async getProjectID(target: string): Promise { + const url = new URL(target); + + if ( + // absPaths to gitlab files should contain /-/blob + // ex: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + !url.pathname.match(/\/\-\/blob\//) + ) { + throw new Error('Please provide full path to yaml file from Gitlab'); + } + try { + const repo = url.pathname.split('/-/blob/')[0]; + + // Find ProjectID from url + // convert 'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath' + // to 'https://gitlab.com/api/v4/projects/groupA%2Fteams%2FsubgroupA%2FteamA%2Frepo' + const repoIDLookup = new URL( + `${url.protocol + url.hostname}/api/v4/projects/${encodeURIComponent( + repo.replace(/^\//, ''), + )}`, + ); + const response = await fetch( + repoIDLookup.toString(), + this.getRequestOptions(), + ); + const projectIDJson = await response.json(); + const projectID: Number = projectIDJson.id; + + return projectID; + } catch (e) { + throw new Error(`Could not get Gitlab ProjectID for: ${target}, ${e}`); + } } }