diff --git a/.changeset/orange-pianos-eat.md b/.changeset/orange-pianos-eat.md new file mode 100644 index 0000000000..5e085a612b --- /dev/null +++ b/.changeset/orange-pianos-eat.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': minor +'@backstage/integration': minor +--- + +Implemented `readTree` for Harness provider to support TechDocs functionality diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index bee03ab45e..a696b850a8 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -70,6 +70,7 @@ See [TechDocs Architecture](architecture.md) to get an overview of where the bel | GitLab Enterprise | Yes ✅ | | Gitea | Yes ✅ | | AWS CodeCommit | Yes ✅ | +| Harness Code | Yes ✅ | ### File storage providers diff --git a/packages/backend-common/api-report.md b/packages/backend-common/api-report.md index 758924b346..2947741fe3 100644 --- a/packages/backend-common/api-report.md +++ b/packages/backend-common/api-report.md @@ -526,13 +526,18 @@ export class GitlabUrlReader implements UrlReader { // @public export class HarnessUrlReader implements UrlReader { - constructor(integration: HarnessIntegration); + constructor( + integration: HarnessIntegration, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); // (undocumented) static factory: ReaderFactory; // (undocumented) read(url: string): Promise; // (undocumented) - readTree(): Promise; + readTree(url: string, options?: ReadTreeOptions): Promise; // (undocumented) readUrl(url: string, options?: ReadUrlOptions): Promise; // (undocumented) diff --git a/packages/backend-common/src/reading/HarnessUrlReader.test.ts b/packages/backend-common/src/reading/HarnessUrlReader.test.ts index 7cc37add3b..f1efbca108 100644 --- a/packages/backend-common/src/reading/HarnessUrlReader.test.ts +++ b/packages/backend-common/src/reading/HarnessUrlReader.test.ts @@ -27,6 +27,9 @@ import { UrlReaderPredicateTuple } from './types'; import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; import { HarnessUrlReader } from './HarnessUrlReader'; +import { NotFoundError, NotModifiedError } from '@backstage/errors'; +import fs from 'fs-extra'; +import path from 'path'; const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), @@ -49,6 +52,7 @@ const harnessProcessor = new HarnessUrlReader( }), ), ), + { treeResponseFactory }, ); const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { @@ -62,6 +66,7 @@ const responseBuffer = Buffer.from('Apache License'); const harnessApiResponse = (content: any) => { return content; }; +const commitHash = '3bdd5457286abdf920db4b77bf2fef79a06190c2'; const handlers = [ rest.get( @@ -95,6 +100,22 @@ const handlers = [ ); }, ), + rest.get( + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName', + (_req, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/json'), + ctx.json({ latest_commit: { sha: commitHash } }), + ); + }, + ), + rest.get( + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName3/projectName/repoName/:path+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName', + (_, res, ctx) => { + return res(ctx.status(404)); + }, + ), ]; describe('HarnessUrlReader', () => { @@ -170,7 +191,7 @@ describe('HarnessUrlReader', () => { 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml', ), ).rejects.toThrow( - 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/404error.yaml?routingId=accountId&git_ref=refMain, 404 Not Found', + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/404error.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/404error.yaml?routingId=accountId&git_ref=refs/heads/refMain, 404 Not Found', ); }); @@ -180,8 +201,60 @@ describe('HarnessUrlReader', () => { 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml', ), ).rejects.toThrow( - 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refMain, 500 Internal Server Error', + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml x https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refs/heads/refMain, 500 Internal Server Error', ); }); }); + + describe('readTree', () => { + const repoBuffer = fs.readFileSync( + path.resolve(__dirname, '__fixtures__/mock-main.zip'), + ); + + it('should be able to get archive', async () => { + worker.use( + rest.get( + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName2/projectName/repoName/:path+/archive/branchName.zip', + (_, res, ctx) => { + return res( + ctx.status(200), + ctx.set('Content-Type', 'application/gzip'), + ctx.set( + 'content-disposition', + 'attachment; filename=backstage-mock.zip', + ), + ctx.body(repoBuffer), + ); + }, + ), + ); + + const response = await harnessProcessor.readTree( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName', + ); + expect(response.etag).toBe(commitHash); + + const files = await response.files(); + expect(files.length).toBe(2); + }); + + it('should return not modified', async () => { + await expect( + harnessProcessor.readTree( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName2/projects/projectName/repos/repoName/files/branchName2', + { + etag: commitHash, + }, + ), + ).rejects.toThrow(NotModifiedError); + }); + + it('should return not found', async () => { + await expect( + harnessProcessor.readTree( + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName3/projects/projectName/repos/repoName/files/branchName3', + ), + ).rejects.toThrow(NotFoundError); + }); + }); }); diff --git a/packages/backend-common/src/reading/HarnessUrlReader.ts b/packages/backend-common/src/reading/HarnessUrlReader.ts index 09291c5176..76ad882d80 100644 --- a/packages/backend-common/src/reading/HarnessUrlReader.ts +++ b/packages/backend-common/src/reading/HarnessUrlReader.ts @@ -18,12 +18,18 @@ import { getHarnessFileContentsUrl, HarnessIntegration, ScmIntegrations, + getHarnessLatestCommitUrl, + getHarnessArchiveUrl, + parseHarnessUrl, } from '@backstage/integration'; -import { ReadUrlOptions, ReadUrlResponse } from './types'; import { ReaderFactory, + ReadTreeOptions, ReadTreeResponse, SearchResponse, + ReadTreeResponseFactory, + ReadUrlOptions, + ReadUrlResponse, UrlReader, } from './types'; import fetch, { Response } from 'node-fetch'; @@ -42,11 +48,13 @@ import { Readable } from 'stream'; * @public */ export class HarnessUrlReader implements UrlReader { - static factory: ReaderFactory = ({ config }) => { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { return ScmIntegrations.fromConfig(config) .harness.list() .map(integration => { - const reader = new HarnessUrlReader(integration); + const reader = new HarnessUrlReader(integration, { + treeResponseFactory, + }); const predicate = (url: URL) => { return url.host === integration.config.host; }; @@ -54,8 +62,12 @@ export class HarnessUrlReader implements UrlReader { }); }; - constructor(private readonly integration: HarnessIntegration) {} - + constructor( + private readonly integration: HarnessIntegration, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ) {} async read(url: string): Promise { const response = await this.readUrl(url); return response.buffer(); @@ -109,8 +121,37 @@ export class HarnessUrlReader implements UrlReader { throw new Error(message); } - readTree(): Promise { - throw new Error('HarnessUrlReader readTree not implemented.'); + async readTree( + url: string, + options?: ReadTreeOptions, + ): Promise { + const lastCommitHash = await this.getLastCommitHash(url); + + if (options?.etag && options.etag === lastCommitHash) { + throw new NotModifiedError(); + } + + const archiveUri = getHarnessArchiveUrl(this.integration.config, url); + + let response: Response; + try { + response = await fetch(archiveUri, { + method: 'GET', + ...getHarnessRequestOptions(this.integration.config), + signal: options?.signal as any, + }); + } catch (e) { + throw new Error(`Unable to read ${archiveUri}, ${e}`); + } + + const parsedUri = parseHarnessUrl(this.integration.config, url); + + return this.deps.treeResponseFactory.fromZipArchive({ + stream: Readable.from(response.body), + subpath: parsedUri.path, + etag: lastCommitHash, + filter: options?.filter, + }); } search(): Promise { throw new Error('HarnessUrlReader search not implemented.'); @@ -122,4 +163,21 @@ export class HarnessUrlReader implements UrlReader { this.integration.config.token || this.integration.config.apiKey, )}}`; } + private async getLastCommitHash(url: string): Promise { + const commitUri = getHarnessLatestCommitUrl(this.integration.config, url); + + const response = await fetch( + commitUri, + getHarnessRequestOptions(this.integration.config), + ); + if (!response.ok) { + const message = `Failed to retrieve latest commit information from ${commitUri}, ${response.status} ${response.statusText}`; + if (response.status === 404) { + throw new NotFoundError(message); + } + throw new Error(message); + } + + return (await response.json()).latest_commit.sha; + } } diff --git a/packages/integration/api-report.md b/packages/integration/api-report.md index 2029dac600..4945847751 100644 --- a/packages/integration/api-report.md +++ b/packages/integration/api-report.md @@ -513,12 +513,24 @@ export function getGitLabRequestOptions(config: GitLabIntegrationConfig): { headers: Record; }; +// @public +export function getHarnessArchiveUrl( + config: HarnessIntegrationConfig, + url: string, +): string; + // @public export function getHarnessFileContentsUrl( config: HarnessIntegrationConfig, url: string, ): string; +// @public +export function getHarnessLatestCommitUrl( + config: HarnessIntegrationConfig, + url: string, +): string; + // @public export function getHarnessRequestOptions(config: HarnessIntegrationConfig): { headers?: Record; @@ -764,6 +776,22 @@ export function parseGiteaUrl( path: string; }; +// @public +export function parseHarnessUrl( + config: HarnessIntegrationConfig, + url: string, +): { + baseUrl: string; + accountId: string; + orgName: string; + projectName: string; + refString: string; + repoName: string; + path: string; + refDashStr: string; + branch: string; +}; + // @public export type PersonalAccessTokenCredential = AzureCredentialBase & { kind: 'PersonalAccessToken'; diff --git a/packages/integration/src/harness/HarnessIntegration.test.ts b/packages/integration/src/harness/HarnessIntegration.test.ts index 2d85204f16..8a936dc07b 100644 --- a/packages/integration/src/harness/HarnessIntegration.test.ts +++ b/packages/integration/src/harness/HarnessIntegration.test.ts @@ -117,7 +117,7 @@ describe('HarnessIntegration', () => { 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/edit/refMain/~/all-apis.yaml', ), ).toBe( - 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/edit/all-apis.yaml', + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repoName/files/refMain/~/all-apis.yaml', ); }); }); diff --git a/packages/integration/src/harness/core.test.ts b/packages/integration/src/harness/core.test.ts index d3d402806a..809c710ecf 100644 --- a/packages/integration/src/harness/core.test.ts +++ b/packages/integration/src/harness/core.test.ts @@ -18,9 +18,12 @@ import { setupServer } from 'msw/node'; import { setupRequestMockHandlers } from '../helpers'; import { HarnessIntegrationConfig } from './config'; import { + getHarnessArchiveUrl, getHarnessEditContentsUrl, getHarnessFileContentsUrl, + getHarnessLatestCommitUrl, getHarnessRequestOptions, + parseHarnessUrl, } from './core'; describe('Harness code core', () => { @@ -38,7 +41,7 @@ describe('Harness code core', () => { 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/files/refMain/~/all-apis.yaml', ), ).toEqual( - 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refMain', + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/raw/all-apis.yaml?routingId=accountId&git_ref=refs/heads/refMain', ); }); }); @@ -54,7 +57,39 @@ describe('Harness code core', () => { 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repos/repoName/edit/refMain/~/all-apis.yaml', ), ).toEqual( - 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projName/repoName/+/edit/all-apis.yaml', + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projName/repoName/files/refMain/~/all-apis.yaml', + ); + }); + }); + + describe('getHarnessArchiveUrl', () => { + it('can create an url from arguments', () => { + const config: HarnessIntegrationConfig = { + host: 'app.harness.io', + }; + expect( + getHarnessArchiveUrl( + config, + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName', + ), + ).toEqual( + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/archive/branchName.zip?routingId=accountId', + ); + }); + }); + + describe('getHarnessLatestCommitUrl', () => { + it('can create an url from arguments', () => { + const config: HarnessIntegrationConfig = { + host: 'app.harness.io', + }; + expect( + getHarnessLatestCommitUrl( + config, + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName', + ), + ).toEqual( + 'https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName', ); }); }); @@ -88,4 +123,50 @@ describe('Harness code core', () => { ).toEqual('a'); }); }); + + describe('parseHarnessUrl', () => { + it('can fetch harness url', () => { + const config: HarnessIntegrationConfig = { + host: 'app.harness.io', + }; + expect( + parseHarnessUrl( + config, + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName', + ), + ).toEqual({ + accountId: 'accountId', + baseUrl: 'https://app.harness.io', + orgName: 'orgName', + path: '', + projectName: 'projectName', + refDashStr: '', + refString: '', + repoName: 'repoName', + branch: 'branchName', + }); + }); + + it('provide path without starting slash', () => { + const config: HarnessIntegrationConfig = { + host: 'app.harness.io', + }; + expect( + parseHarnessUrl( + config, + 'https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName/simple/~/path', + ), + ).toEqual({ + accountId: 'accountId', + baseUrl: 'https://app.harness.io', + orgName: 'orgName', + path: 'path', + projectName: 'projectName', + refDashStr: 'branchName-simple', + refString: 'branchName/simple', + repoName: 'repoName', + branch: 'branchName/simple', + }); + }); + }); }); diff --git a/packages/integration/src/harness/core.ts b/packages/integration/src/harness/core.ts index 3a61905e27..7faa6b6d9c 100644 --- a/packages/integration/src/harness/core.ts +++ b/packages/integration/src/harness/core.ts @@ -33,36 +33,13 @@ export function getHarnessEditContentsUrl( config: HarnessIntegrationConfig, url: string, ) { - try { - const baseUrl = `https://${config.host}`; - const [ - _blank, - _ng, - _account, - accountId, - _module, - _moduleName, - _org, - orgName, - _projects, - projectName, - _repos, - repoName, - _files, - _ref, - _branch, - ...path - ] = url.replace(baseUrl, '').split('/'); - const pathWithoutSlash = path.join('/').replace(/^\//, ''); - return `${baseUrl}/gateway/code/api/v1/repos/${accountId}/${orgName}/${projectName}/${repoName}/+/edit/${pathWithoutSlash}`; - } catch (e) { - throw new Error(`Incorrect URL: ${url}, ${e}`); - } + const parsedUrl = parseHarnessUrl(config, url); + return `${parsedUrl.baseUrl}/ng/account/${parsedUrl.accountId}/module/code/orgs/${parsedUrl.orgName}/projects/${parsedUrl.projectName}/${parsedUrl.repoName}/files/${parsedUrl.branch}/~/${parsedUrl.path}`; } /** * Given a file path URL, - * it returns an API URL which returns the contents of the file. + * it returns an API URL which returns the contents of the file . * @remarks * * Converts @@ -77,35 +54,52 @@ export function getHarnessFileContentsUrl( config: HarnessIntegrationConfig, url: string, ) { - try { - const baseUrl = `https://${config.host}`; - const [ - _blank, - _ng, - _account, - accountId, - _module, - _moduleName, - _org, - orgName, - _projects, - projectName, - _repos, - repoName, - _files, - _ref, - _branch, - ...path - ] = url.replace(baseUrl, '').split('/'); - const urlParts = url.replace(baseUrl, '').split('/'); - const refAndPath = urlParts.slice(13); - const refIndex = refAndPath.findIndex(item => item === '~'); - const refString = refAndPath.slice(0, refIndex); - const pathWithoutSlash = path.join('/').replace(/^\//, ''); - return `${baseUrl}/gateway/code/api/v1/repos/${accountId}/${orgName}/${projectName}/${repoName}/+/raw/${pathWithoutSlash}?routingId=${accountId}&git_ref=${refString}`; - } catch (e) { - throw new Error(`Incorrect URL: ${url}, ${e}`); - } + const parsedUrl = parseHarnessUrl(config, url); + return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/raw/${parsedUrl.path}?routingId=${parsedUrl.accountId}&git_ref=refs/heads/${parsedUrl.refString}`; +} + +/** + * Given a URL pointing to a repository/path, returns a URL + * for archive contents of the repository. + * + * @remarks + * + * Converts + * from: https://qa.harness.io/ng/account/accountId/module/code/orgs/orgId/projects/projectName/repos/repoName/files/branch/~/fileName + * to: https://qa.harness.io/gateway/code/api/v1/repos/accountId/orgId/projectName/repoName/+/archive/branch.zip?routingId=accountId + * + * @param url - A URL pointing to a repository/path + * @param config - The relevant provider config + * @public + */ +export function getHarnessArchiveUrl( + config: HarnessIntegrationConfig, + url: string, +) { + const parsedUrl = parseHarnessUrl(config, url); + return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/archive/${parsedUrl.branch}.zip?routingId=${parsedUrl.accountId}`; +} + +/** + * Given a URL pointing to a repository branch, returns a URL + * for latest commit information. + * + * @remarks + * + * Converts + * from: https://app.harness.io/ng/account/accountId/module/code/orgs/orgName/projects/projectName/repos/repoName/files/branchName + * to: https://app.harness.io/gateway/code/api/v1/repos/accountId/orgName/projectName/repoName/+/content?routingId=accountId&include_commit=true&git_ref=refs/heads/branchName + * + * @param url - A URL pointing to a repository branch + * @param config - The relevant provider config + * @public + */ +export function getHarnessLatestCommitUrl( + config: HarnessIntegrationConfig, + url: string, +) { + const parsedUrl = parseHarnessUrl(config, url); + return `${parsedUrl.baseUrl}/gateway/code/api/v1/repos/${parsedUrl.accountId}/${parsedUrl.orgName}/${parsedUrl.projectName}/${parsedUrl.repoName}/+/content?routingId=${parsedUrl.accountId}&include_commit=true&git_ref=refs/heads/${parsedUrl.branch}`; } /** @@ -130,3 +124,79 @@ export function getHarnessRequestOptions(config: HarnessIntegrationConfig): { headers, }; } + +/** + * Return parsed git url properties. + * + * @param config - A Harness provider config + * @param url - A URL pointing to a repository + * @public + */ +export function parseHarnessUrl( + config: HarnessIntegrationConfig, + url: string, +): { + baseUrl: string; + accountId: string; + orgName: string; + projectName: string; + refString: string; + repoName: string; + path: string; + refDashStr: string; + branch: string; +} { + const baseUrl = `https://${config.host}`; + try { + const pathUrl = new URL(url); + const pathSegments = pathUrl.pathname + .split('/') + .filter(segment => segment !== ''); + const urlParts = pathUrl.pathname.split('/'); + const [ + _ng, + _account, + accountId, + _module, + _moduleName, + _org, + orgName, + _projects, + projectName, + _repos, + repoName, + _files, + _ref, + _branch, + ..._path + ] = pathSegments; + const refAndPath = urlParts.slice( + urlParts.findIndex(i => i === 'files' || i === 'edit') + 1, + ); + const refIndex = refAndPath.findIndex(item => item === '~'); + const refString = refAndPath.slice(0, refIndex).join('/'); + const pathWithoutSlash = + refIndex !== -1 + ? refAndPath + .slice(refIndex + 1) + .join('/') + .replace(/^\//, '') + : ''; + return { + baseUrl: baseUrl, + accountId: accountId, + orgName: orgName, + projectName: projectName, + refString: refString, + path: pathWithoutSlash, + repoName: repoName, + refDashStr: refAndPath.slice(0, refIndex).join('-'), + branch: + refIndex !== -1 + ? refAndPath.slice(0, refIndex).join('/') + : refAndPath.join('/'), + }; + } catch (e) { + throw new Error(`Incorrect URL: ${url}, ${e}`); + } +} diff --git a/packages/integration/src/harness/index.ts b/packages/integration/src/harness/index.ts index 264df657f0..65cd73e49b 100644 --- a/packages/integration/src/harness/index.ts +++ b/packages/integration/src/harness/index.ts @@ -14,6 +14,12 @@ * limitations under the License. */ export { HarnessIntegration } from './HarnessIntegration'; -export { getHarnessRequestOptions, getHarnessFileContentsUrl } from './core'; +export { + getHarnessRequestOptions, + getHarnessFileContentsUrl, + getHarnessArchiveUrl, + getHarnessLatestCommitUrl, + parseHarnessUrl, +} from './core'; export { readHarnessConfig } from './config'; export type { HarnessIntegrationConfig } from './config';