paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 50 Packages Projects Releases Wiki Activity

Introduce the annotation backstage.io/techdocs-ref: <relative-target> as an alias for backstage.io/techdocs-ref: dir:<relative-target>

Browse Source 
Signed-off-by: Dominik Henneke <dominik.henneke@sda-se.com>
This commit is contained in:
Dominik Henneke
2021-07-09 09:54:42 +02:00
parent ffae1bb6e4
commit d32d01e5bc
12 changed files with 556 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/api-report.md
+36 -3
View File
@@ -15,6 +15,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
import { GitLabIntegrationConfig } from '@backstage/integration';
import { Logger as Logger_2 } from 'winston';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { ScmIntegrationRegistry } from '@backstage/integration';
import { UrlReader } from '@backstage/backend-common';
import { Writable } from 'stream';
@@ -47,9 +48,15 @@ export class CommonGitPreparer implements PreparerBase {
//
// @public (undocumented)
export class DirectoryPreparer implements PreparerBase {
constructor(config: Config, logger: Logger_2, reader: UrlReader);
constructor(config: Config, _logger: Logger_2, reader: UrlReader);
// (undocumented)
prepare(entity: Entity): Promise<PreparerResponse>;
prepare(
entity: Entity,
options?: {
logger?: Logger_2;
etag?: string;
},
): Promise<PreparerResponse>;
}
// Warning: (ae-missing-release-tag) "GeneratorBase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -130,6 +137,18 @@ export const getDefaultBranch: (
config: Config,
) => Promise<string>;
// Warning: (ae-missing-release-tag) "getDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export const getDirLocation: (
entity: Entity,
) =>
| {
type: 'dir';
target: string;
}
| undefined;
// Warning: (ae-missing-release-tag) "getDocFilesFromRepository" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
@@ -189,7 +208,10 @@ export const getLastCommitTimestamp: (
// Warning: (ae-missing-release-tag) "getLocationForEntity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export const getLocationForEntity: (entity: Entity) => ParsedLocationAnnotation;
export const getLocationForEntity: (
entity: Entity,
scmIntegration: ScmIntegrationRegistry,
) => ParsedLocationAnnotation;
// Warning: (ae-missing-release-tag) "getTokenForGitRepo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
@@ -355,6 +377,17 @@ export type TechDocsMetadata = {
etag: string;
};
// Warning: (ae-missing-release-tag) "transformDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export const transformDirLocation: (
entity: Entity,
scmIntegrations: ScmIntegrationRegistry,
) => {
type: 'dir' | 'url';
target: string;
};
// Warning: (ae-missing-release-tag) "UrlPreparer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
packages/techdocs-common/src/helpers.test.ts
+238 -1
View File
@@ -19,14 +19,27 @@ import {
SearchResponse,
UrlReader,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { Entity, getEntitySourceLocation } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import os from 'os';
import path from 'path';
import { Readable } from 'stream';
import {
getDirLocation,
getDocFilesFromRepository,
getLocationForEntity,
parseReferenceAnnotation,
transformDirLocation,
} from './helpers';
jest.mock('@backstage/catalog-model', () => ({
...jest.requireActual('@backstage/catalog-model'),
getEntitySourceLocation: jest.fn(),
}));
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const entityBase: Entity = {
metadata: {
namespace: 'default',
@@ -81,6 +94,10 @@ const mockEntityWithBadAnnotation: Entity = {
},
};
const scmIntegrations = ScmIntegrations.fromConfig(new ConfigReader({}));
afterEach(() => jest.resetAllMocks());
describe('parseReferenceAnnotation', () => {
it('should parse annotation', () => {
const parsedLocationAnnotation = parseReferenceAnnotation(
@@ -109,10 +126,230 @@ describe('parseReferenceAnnotation', () => {
});
});
describe('getDirLocation', () => {
it.each`
techdocsRef | responseTarget
${undefined} | ${undefined}
${16} | ${undefined}
${'.'} | ${'.'}
${'dir:.'} | ${'.'}
${'./relative'} | ${'./relative'}
${'dir:./relative'} | ${'./relative'}
${'dir:https://github.com...'} | ${'https://github.com...'}
${'url:https://github.com...'} | ${undefined}
`(
'should handle "backstage.io/techdocs-ref: $techdocsRef" correctly',
({ techdocsRef, responseTarget }) => {
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': techdocsRef,
},
},
};
const result = getDirLocation(entity);
expect(result).toEqual(
responseTarget && { type: 'dir', target: responseTarget },
);
},
);
it('Reject https urls and hint to the url: location type', async () => {
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': 'https://github.com/backstage/backstage',
},
},
};
expect(() => getDirLocation(entity)).toThrow(
/please prefix it with 'url:'/,
);
});
});
describe('transformDirLocation', () => {
it('should reject missing annotation', () => {
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
},
};
expect(() => transformDirLocation(entity, scmIntegrations)).toThrow(
/No techdocs location annotation provided in entity: component:default\/test/,
);
});
it.each`
techdocsRef | target
${'.'} | ${'https://my-url/folder/'}
${'./sub-folder'} | ${'https://my-url/folder/sub-folder'}
`(
'should transform "$techdocsRef" for url type locations',
({ techdocsRef, target }) => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'url',
target: 'https://my-url/folder/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': techdocsRef,
},
},
};
const result = transformDirLocation(entity, scmIntegrations);
expect(result).toEqual({ type: 'url', target });
},
);
it.each`
techdocsRef | target
${'.'} | ${path.join(rootDir, 'working-copy')}
${'./sub-folder'} | ${path.join(rootDir, 'working-copy', 'sub-folder')}
`(
'should transform "$techdocsRef" for file type locations',
({ techdocsRef, target }) => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'file',
target: path.join(rootDir, 'working-copy', 'catalog-info.yaml'),
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': techdocsRef,
},
},
};
const result = transformDirLocation(entity, scmIntegrations);
expect(result).toEqual({ type: 'dir', target });
},
);
it('should reject unsafe file location', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'file',
target: '/tmp/catalog-info.yaml',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': '..',
},
},
};
expect(() => transformDirLocation(entity, scmIntegrations)).toThrow(
/Relative path is not allowed to refer to a directory outside its parent/,
);
});
it('should reject other location types', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'other',
target: '/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': '.',
},
},
};
expect(() => transformDirLocation(entity, scmIntegrations)).toThrow(
/Unable to resolve location type other/,
);
});
});
describe('getLocationForEntity', () => {
it('should handle implicit dir locations', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'url',
target: 'https://my-url/folder/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': '.',
},
},
};
const parsedLocationAnnotation = getLocationForEntity(
entity,
scmIntegrations,
);
expect(parsedLocationAnnotation.type).toBe('url');
expect(parsedLocationAnnotation.target).toBe('https://my-url/folder/');
});
it('should handle dir locations', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'url',
target: 'https://my-url/folder/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': 'dir:.',
},
},
};
const parsedLocationAnnotation = getLocationForEntity(
entity,
scmIntegrations,
);
expect(parsedLocationAnnotation.type).toBe('url');
expect(parsedLocationAnnotation.target).toBe('https://my-url/folder/');
});
it('should get location for entity', () => {
const parsedLocationAnnotation = getLocationForEntity(
mockEntityWithAnnotation,
scmIntegrations,
);
expect(parsedLocationAnnotation.type).toBe('url');
expect(parsedLocationAnnotation.target).toBe(
packages/techdocs-common/src/helpers.ts
+117 -11
View File
@@ -14,10 +14,20 @@
* limitations under the License.
*/
import { Git, UrlReader } from '@backstage/backend-common';
import { InputError } from '@backstage/errors';
import { Entity, parseLocationReference } from '@backstage/catalog-model';
import {
Git,
resolveSafeChildPath,
UrlReader,
} from '@backstage/backend-common';
import {
Entity,
getEntitySourceLocation,
parseLocationReference,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { ScmIntegrationRegistry } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
@@ -50,9 +60,113 @@ export const parseReferenceAnnotation = (
};
};
/**
* Check if the entity provides a `backstage.io/techdocs-ref` annotation of type `dir`
* and return the annotation. It accepts the two variants `<target>` and `dir:<target>`.
*
* @param entity - the entity to check
*/
export const getDirLocation = (
entity: Entity,
): { type: 'dir'; target: string } | undefined => {
const annotation = entity.metadata.annotations?.['backstage.io/techdocs-ref'];
if (!annotation) {
return undefined;
}
if (typeof annotation !== 'string') {
return undefined;
}
// if the string doesn't contain `:`, interpret it as the target of a dir type
if (!annotation.includes(':')) {
return { type: 'dir', target: annotation };
}
// note that `backstage.io/techdocs-ref: https://...` is invalid and will throw
const reference = parseLocationReference(annotation);
if (reference.type === 'dir') {
return {
type: 'dir',
target: reference.target,
};
}
// ignore any other types
return undefined;
};
/**
* TechDocs references of type `dir` are relative the source location of the entity.
* This function transforms relative references to absolute ones, based on the
* location the entity was ingested from. If the entity was registered by a `url`
* location, it returns a `url` location with a resolved target that points to the
* targeted subfolder. If the entity was registered by a `file` location, it returns
* an absolute `dir` location.
*
* @param entity - the entity with annotations
* @param scmIntegrations - access to the scmIntegrationt to do url transformations
* @throws if the entity doesn't specify a `dir` location or is ingested from an unsupported location.
* @returns the transformed location with an absolute target.
*/
export const transformDirLocation = (
entity: Entity,
scmIntegrations: ScmIntegrationRegistry,
): { type: 'dir' | 'url'; target: string } => {
const dirLocation = getDirLocation(entity);
if (!dirLocation) {
throw new InputError(
`No techdocs location annotation provided in entity: ${stringifyEntityRef(
entity,
)}`,
);
}
const location = getEntitySourceLocation(entity);
switch (location.type) {
case 'url': {
const target = scmIntegrations.resolveUrl({
url: dirLocation.target,
base: location.target,
});
return {
type: 'url',
target,
};
}
case 'file': {
// only permit targets in the same folder as the target of the `file` location!
const target = resolveSafeChildPath(
path.dirname(location.target),
dirLocation.target,
);
return {
type: 'dir',
target,
};
}
default:
throw new InputError(`Unable to resolve location type ${location.type}`);
}
};
export const getLocationForEntity = (
entity: Entity,
scmIntegration: ScmIntegrationRegistry,
): ParsedLocationAnnotation => {
// try to resolve relative references first
if (getDirLocation(entity) !== undefined) {
return transformDirLocation(entity, scmIntegration);
}
const { type, target } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
@@ -64,14 +178,6 @@ export const getLocationForEntity = (
case 'azure/api':
case 'url':
return { type, target };
case 'dir':
if (path.isAbsolute(target)) {
return { type, target };
}
return parseReferenceAnnotation(
'backstage.io/managed-by-location',
entity,
);
default:
throw new Error(`Invalid reference annotation ${type}`);
}
packages/techdocs-common/src/stages/prepare/dir.test.ts
+7 -11
View File
@@ -15,7 +15,6 @@
*/
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { checkoutGitRepository } from '../../helpers';
import { DirectoryPreparer } from './dir';
function normalizePath(path: string) {
@@ -27,8 +26,6 @@ function normalizePath(path: string) {
jest.mock('../../helpers', () => ({
...jest.requireActual<{}>('../../helpers'),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
getLastCommitTimestamp: jest.fn(() => 12345678),
}));
const logger = getVoidLogger();
@@ -71,7 +68,7 @@ describe('directory preparer', () => {
expect(normalizePath(preparedDir)).toEqual('/directory/our-documentation');
});
it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
it('should reject when techdocs-ref is absolute', async () => {
const directoryPreparer = new DirectoryPreparer(
mockConfig,
logger,
@@ -84,11 +81,12 @@ describe('directory preparer', () => {
'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
});
const { preparedDir } = await directoryPreparer.prepare(mockEntity);
expect(normalizePath(preparedDir)).toEqual('/our-documentation/techdocs');
await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
/Relative path is not allowed to refer to a directory outside its parent/,
);
});
it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
it('should reject when managed-by-location is a git repository', async () => {
const directoryPreparer = new DirectoryPreparer(
mockConfig,
logger,
@@ -101,10 +99,8 @@ describe('directory preparer', () => {
'backstage.io/techdocs-ref': 'dir:./docs',
});
const { preparedDir } = await directoryPreparer.prepare(mockEntity);
expect(normalizePath(preparedDir)).toEqual(
'/tmp/backstage-repo/org/name/branch/docs',
await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
/Unable to resolve location type github/,
);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
});
});
packages/techdocs-common/src/stages/prepare/dir.ts
+23 -70
View File
@@ -15,104 +15,57 @@
*/
import { UrlReader } from '@backstage/backend-common';
import { InputError, NotModifiedError } from '@backstage/errors';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import parseGitUrl from 'git-url-parse';
import path from 'path';
import { Logger } from 'winston';
import { InputError } from '@backstage/errors';
import {
checkoutGitRepository,
getLastCommitTimestamp,
parseReferenceAnnotation,
} from '../../helpers';
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import { Logger } from 'winston';
import { transformDirLocation } from '../../helpers';
import { PreparerBase, PreparerResponse } from './types';
export class DirectoryPreparer implements PreparerBase {
constructor(
private readonly config: Config,
private readonly logger: Logger,
private readonly reader: UrlReader,
) {
this.config = config;
this.logger = logger;
private readonly scmIntegrations: ScmIntegrationRegistry;
private readonly reader: UrlReader;
constructor(config: Config, _logger: Logger, reader: UrlReader) {
this.reader = reader;
this.scmIntegrations = ScmIntegrations.fromConfig(config);
}
private async resolveManagedByLocationToDir(
async prepare(
entity: Entity,
options?: { etag?: string },
options?: { logger?: Logger; etag?: string },
): Promise<PreparerResponse> {
const { type, target } = parseReferenceAnnotation(
'backstage.io/managed-by-location',
entity,
);
const { type, target } = transformDirLocation(entity, this.scmIntegrations);
this.logger.debug(
`Building docs for entity with type 'dir' and managed-by-location '${type}'`,
);
switch (type) {
case 'url': {
options?.logger?.info(`Download documentation from ${target}`);
// the target is an absolute url since it has already been transformed
const response = await this.reader.readTree(target, {
etag: options?.etag,
});
const preparedDir = await response.dir();
return {
preparedDir,
preparedDir: await response.dir(),
etag: response.etag,
};
}
case 'github':
case 'gitlab':
case 'azure/api': {
const parsedGitLocation = parseGitUrl(target);
const repoLocation = await checkoutGitRepository(
target,
this.config,
this.logger,
);
// Check if etag has changed for cache invalidation.
const etag = await getLastCommitTimestamp(repoLocation, this.logger);
if (options?.etag === etag.toString()) {
throw new NotModifiedError();
}
case 'dir': {
return {
preparedDir: path.dirname(
path.join(repoLocation, parsedGitLocation.filepath),
),
etag: etag.toString(),
};
}
case 'file':
return {
preparedDir: path.dirname(target),
// the transformation already validated that the target is in a safe location
preparedDir: target,
// Instead of supporting caching on local sources, use techdocs-cli for local development and debugging.
etag: '',
};
}
default:
throw new InputError(`Unable to resolve location type ${type}`);
}
}
async prepare(entity: Entity): Promise<PreparerResponse> {
this.logger.warn(
'You are using the legacy dir preparer in TechDocs which will be removed in near future (March 2021). ' +
'Migrate to URL reader by updating `backstage.io/techdocs-ref` annotation in `catalog-info.yaml` ' +
'to be prefixed with `url:`. Read the migration guide and benefits at https://github.com/backstage/backstage/issues/4409 ',
);
const { target } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
// This will throw NotModified error if etag has not changed.
const response = await this.resolveManagedByLocationToDir(entity);
return {
preparedDir: path.resolve(response.preparedDir, target),
etag: response.etag,
};
}
}
packages/techdocs-common/src/stages/prepare/preparers.ts
+21 -9
View File
@@ -13,15 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { UrlReader } from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { Entity, parseLocationReference } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { Logger } from 'winston';
import { parseReferenceAnnotation } from '../../helpers';
import { DirectoryPreparer } from './dir';
import { getDirLocation } from '../../helpers';
import { CommonGitPreparer } from './commonGit';
import { UrlPreparer } from './url';
import { DirectoryPreparer } from './dir';
import { PreparerBase, PreparerBuilder, RemoteProtocol } from './types';
import { UrlPreparer } from './url';
type factoryOptions = {
logger: Logger;
@@ -61,11 +63,21 @@ export class Preparers implements PreparerBuilder {
}
get(entity: Entity): PreparerBase {
const { type } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
const preparer = this.preparerMap.get(type);
const annotation =
entity.metadata.annotations?.['backstage.io/techdocs-ref'];
if (!annotation) {
throw new InputError(
`No location annotation provided in entity: ${entity.metadata.name}`,
);
}
// the dir processor handles both `<target>` and `dir:<target>`
if (getDirLocation(entity) !== undefined) {
return this.preparerMap.get('dir')!;
}
const { type } = parseLocationReference(annotation);
const preparer = this.preparerMap.get(type as RemoteProtocol);
if (!preparer) {
throw new Error(`No preparer registered for type: "${type}"`);