Merge pull request #6495 from SDA-SE/feat/relative-ref-tmp

[TechDocs] Introduce the annotation `backstage.io/techdocs-ref: <relative-target>` as an alias for `backstage.io/techdocs-ref: dir:<relative-target>`
This commit is contained in:
Otto Sichert
2021-07-21 16:41:23 +02:00
committed by GitHub
11 changed files with 420 additions and 110 deletions
+88
View File
@@ -0,0 +1,88 @@
---
'@backstage/techdocs-common': minor
'@backstage/plugin-techdocs-backend': minor
---
Improve the annotation `backstage.io/techdocs-ref: dir:<relative-target>` that links to a path that is relative to the source of the annotated entity.
This annotation works with the basic and the recommended flow, however, it will be most useful with the basic approach.
This change remove the deprecation of the `dir` reference and provides first-class support for it.
In addition, this change removes the support of the deprecated `github`, `gitlab`, and `azure/api` locations from the `dir` reference preparer.
#### Example Usage
The annotation is convenient if the documentation is stored in the same location, i.e. the same git repository, as the `catalog-info.yaml`.
While it is still supported to add full URLs such as `backstage.io/techdocs-ref: url:https://...` for custom setups, documentation is mostly stored in the same repository as the entity definition.
By automatically resolving the target relative to the registration location of the entity, the configuration overhead for this default setup is minimized.
Since it leverages the `@backstage/integrations` package for the URL resolution, this is compatible with every supported source.
Consider the following examples:
1. "I have a repository with a single `catalog-info.yaml` and a TechDocs page in the root folder!"
```
https://github.com/backstage/example/tree/main/
|- catalog-info.yaml
| > apiVersion: backstage.io/v1alpha1
| > kind: Component
| > metadata:
| > name: example
| > annotations:
| > backstage.io/techdocs-ref: dir:. # -> same folder
| > spec: {}
|- docs/
|- mkdocs.yml
```
2. "I have a repository with a single `catalog-info.yaml` and my TechDocs page in located in a folder!"
```
https://bitbucket.org/my-owner/my-project/src/master/
|- catalog-info.yaml
| > apiVersion: backstage.io/v1alpha1
| > kind: Component
| > metadata:
| > name: example
| > annotations:
| > backstage.io/techdocs-ref: dir:./some-folder # -> subfolder
| > spec: {}
|- some-folder/
|- docs/
|- mkdocs.yml
```
3. "I have a mono repository that hosts multiple components!"
```
https://dev.azure.com/organization/project/_git/repository
|- my-1st-module/
|- catalog-info.yaml
| > apiVersion: backstage.io/v1alpha1
| > kind: Component
| > metadata:
| > name: my-1st-module
| > annotations:
| > backstage.io/techdocs-ref: dir:. # -> same folder
| > spec: {}
|- docs/
|- mkdocs.yml
|- my-2nd-module/
|- catalog-info.yaml
| > apiVersion: backstage.io/v1alpha1
| > kind: Component
| > metadata:
| > name: my-2nd-module
| > annotations:
| > backstage.io/techdocs-ref: dir:. # -> same folder
| > spec: {}
|- docs/
|- mkdocs.yml
|- catalog-info.yaml
| > apiVersion: backstage.io/v1alpha1
| > kind: Location
| > metadata:
| > name: example
| > spec:
| > targets:
| > - ./*/catalog-info.yaml
```
+25 -3
View File
@@ -15,6 +15,7 @@ import { GitHubIntegrationConfig } from '@backstage/integration';
import { GitLabIntegrationConfig } from '@backstage/integration';
import { Logger as Logger_2 } from 'winston';
import { PluginEndpointDiscovery } from '@backstage/backend-common';
import { ScmIntegrationRegistry } from '@backstage/integration';
import { UrlReader } from '@backstage/backend-common';
import { Writable } from 'stream';
@@ -47,9 +48,15 @@ export class CommonGitPreparer implements PreparerBase {
//
// @public (undocumented)
export class DirectoryPreparer implements PreparerBase {
constructor(config: Config, logger: Logger_2, reader: UrlReader);
constructor(config: Config, _logger: Logger_2, reader: UrlReader);
// (undocumented)
prepare(entity: Entity): Promise<PreparerResponse>;
prepare(
entity: Entity,
options?: {
logger?: Logger_2;
etag?: string;
},
): Promise<PreparerResponse>;
}
// Warning: (ae-missing-release-tag) "GeneratorBase" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
@@ -189,7 +196,10 @@ export const getLastCommitTimestamp: (
// Warning: (ae-missing-release-tag) "getLocationForEntity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export const getLocationForEntity: (entity: Entity) => ParsedLocationAnnotation;
export const getLocationForEntity: (
entity: Entity,
scmIntegration: ScmIntegrationRegistry,
) => ParsedLocationAnnotation;
// Warning: (ae-missing-release-tag) "getTokenForGitRepo" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
@@ -355,6 +365,18 @@ export type TechDocsMetadata = {
etag: string;
};
// Warning: (ae-missing-release-tag) "transformDirLocation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public
export const transformDirLocation: (
entity: Entity,
dirAnnotation: ParsedLocationAnnotation,
scmIntegrations: ScmIntegrationRegistry,
) => {
type: 'dir' | 'url';
target: string;
};
// Warning: (ae-missing-release-tag) "UrlPreparer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
+165 -1
View File
@@ -19,14 +19,26 @@ import {
SearchResponse,
UrlReader,
} from '@backstage/backend-common';
import { Entity } from '@backstage/catalog-model';
import { Entity, getEntitySourceLocation } from '@backstage/catalog-model';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import os from 'os';
import path from 'path';
import { Readable } from 'stream';
import {
getDocFilesFromRepository,
getLocationForEntity,
parseReferenceAnnotation,
transformDirLocation,
} from './helpers';
jest.mock('@backstage/catalog-model', () => ({
...jest.requireActual('@backstage/catalog-model'),
getEntitySourceLocation: jest.fn(),
}));
const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
const entityBase: Entity = {
metadata: {
namespace: 'default',
@@ -81,6 +93,10 @@ const mockEntityWithBadAnnotation: Entity = {
},
};
const scmIntegrations = ScmIntegrations.fromConfig(new ConfigReader({}));
afterEach(() => jest.resetAllMocks());
describe('parseReferenceAnnotation', () => {
it('should parse annotation', () => {
const parsedLocationAnnotation = parseReferenceAnnotation(
@@ -109,10 +125,158 @@ describe('parseReferenceAnnotation', () => {
});
});
describe('transformDirLocation', () => {
it.each`
techdocsRef | target
${'dir:.'} | ${'https://my-url/folder/'}
${'dir:./sub-folder'} | ${'https://my-url/folder/sub-folder'}
`(
'should transform "$techdocsRef" for url type locations',
({ techdocsRef, target }) => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'url',
target: 'https://my-url/folder/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': techdocsRef,
},
},
};
const result = transformDirLocation(
entity,
parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
scmIntegrations,
);
expect(result).toEqual({ type: 'url', target });
},
);
it.each`
techdocsRef | target
${'dir:.'} | ${path.join(rootDir, 'working-copy')}
${'dir:./sub-folder'} | ${path.join(rootDir, 'working-copy', 'sub-folder')}
`(
'should transform "$techdocsRef" for file type locations',
({ techdocsRef, target }) => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'file',
target: path.join(rootDir, 'working-copy', 'catalog-info.yaml'),
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': techdocsRef,
},
},
};
const result = transformDirLocation(
entity,
parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
scmIntegrations,
);
expect(result).toEqual({ type: 'dir', target });
},
);
it('should reject unsafe file location', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'file',
target: '/tmp/catalog-info.yaml',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': 'dir:..',
},
},
};
expect(() =>
transformDirLocation(
entity,
parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
scmIntegrations,
),
).toThrow(
/Relative path is not allowed to refer to a directory outside its parent/,
);
});
it('should reject other location types', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'other',
target: '/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': 'dir:.',
},
},
};
expect(() =>
transformDirLocation(
entity,
parseReferenceAnnotation('backstage.io/techdocs-ref', entity),
scmIntegrations,
),
).toThrow(/Unable to resolve location type other/);
});
});
describe('getLocationForEntity', () => {
it('should handle dir locations', () => {
(getEntitySourceLocation as jest.Mock).mockReturnValue({
type: 'url',
target: 'https://my-url/folder/',
});
const entity = {
apiVersion: '1',
kind: 'Component',
metadata: {
name: 'test',
annotations: {
'backstage.io/techdocs-ref': 'dir:.',
},
},
};
const parsedLocationAnnotation = getLocationForEntity(
entity,
scmIntegrations,
);
expect(parsedLocationAnnotation.type).toBe('url');
expect(parsedLocationAnnotation.target).toBe('https://my-url/folder/');
});
it('should get location for entity', () => {
const parsedLocationAnnotation = getLocationForEntity(
mockEntityWithAnnotation,
scmIntegrations,
);
expect(parsedLocationAnnotation.type).toBe('url');
expect(parsedLocationAnnotation.target).toBe(
+70 -14
View File
@@ -14,10 +14,19 @@
* limitations under the License.
*/
import { Git, UrlReader } from '@backstage/backend-common';
import { InputError } from '@backstage/errors';
import { Entity, parseLocationReference } from '@backstage/catalog-model';
import {
Git,
resolveSafeChildPath,
UrlReader,
} from '@backstage/backend-common';
import {
Entity,
getEntitySourceLocation,
parseLocationReference,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { ScmIntegrationRegistry } from '@backstage/integration';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import os from 'os';
@@ -50,30 +59,77 @@ export const parseReferenceAnnotation = (
};
};
/**
* TechDocs references of type `dir` are relative the source location of the entity.
* This function transforms relative references to absolute ones, based on the
* location the entity was ingested from. If the entity was registered by a `url`
* location, it returns a `url` location with a resolved target that points to the
* targeted subfolder. If the entity was registered by a `file` location, it returns
* an absolute `dir` location.
*
* @param entity - the entity with annotations
* @param dirAnnotation - the parsed techdocs-ref annotation of type 'dir'
* @param scmIntegrations - access to the scmIntegration to do url transformations
* @throws if the entity doesn't specify a `dir` location or is ingested from an unsupported location.
* @returns the transformed location with an absolute target.
*/
export const transformDirLocation = (
entity: Entity,
dirAnnotation: ParsedLocationAnnotation,
scmIntegrations: ScmIntegrationRegistry,
): { type: 'dir' | 'url'; target: string } => {
const location = getEntitySourceLocation(entity);
switch (location.type) {
case 'url': {
const target = scmIntegrations.resolveUrl({
url: dirAnnotation.target,
base: location.target,
});
return {
type: 'url',
target,
};
}
case 'file': {
// only permit targets in the same folder as the target of the `file` location!
const target = resolveSafeChildPath(
path.dirname(location.target),
dirAnnotation.target,
);
return {
type: 'dir',
target,
};
}
default:
throw new InputError(`Unable to resolve location type ${location.type}`);
}
};
export const getLocationForEntity = (
entity: Entity,
scmIntegration: ScmIntegrationRegistry,
): ParsedLocationAnnotation => {
const { type, target } = parseReferenceAnnotation(
const annotation = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
switch (type) {
switch (annotation.type) {
case 'github':
case 'gitlab':
case 'azure/api':
case 'url':
return { type, target };
return annotation;
case 'dir':
if (path.isAbsolute(target)) {
return { type, target };
}
return parseReferenceAnnotation(
'backstage.io/managed-by-location',
entity,
);
return transformDirLocation(entity, annotation, scmIntegration);
default:
throw new Error(`Invalid reference annotation ${type}`);
throw new Error(`Invalid reference annotation ${annotation.type}`);
}
};
@@ -15,7 +15,6 @@
*/
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { checkoutGitRepository } from '../../helpers';
import { DirectoryPreparer } from './dir';
function normalizePath(path: string) {
@@ -27,8 +26,6 @@ function normalizePath(path: string) {
jest.mock('../../helpers', () => ({
...jest.requireActual<{}>('../../helpers'),
checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'),
getLastCommitTimestamp: jest.fn(() => 12345678),
}));
const logger = getVoidLogger();
@@ -71,7 +68,7 @@ describe('directory preparer', () => {
expect(normalizePath(preparedDir)).toEqual('/directory/our-documentation');
});
it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => {
it('should reject when techdocs-ref is absolute', async () => {
const directoryPreparer = new DirectoryPreparer(
mockConfig,
logger,
@@ -84,11 +81,12 @@ describe('directory preparer', () => {
'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs',
});
const { preparedDir } = await directoryPreparer.prepare(mockEntity);
expect(normalizePath(preparedDir)).toEqual('/our-documentation/techdocs');
await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
/Relative path is not allowed to refer to a directory outside its parent/,
);
});
it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => {
it('should reject when managed-by-location is a git repository', async () => {
const directoryPreparer = new DirectoryPreparer(
mockConfig,
logger,
@@ -101,10 +99,8 @@ describe('directory preparer', () => {
'backstage.io/techdocs-ref': 'dir:./docs',
});
const { preparedDir } = await directoryPreparer.prepare(mockEntity);
expect(normalizePath(preparedDir)).toEqual(
'/tmp/backstage-repo/org/name/branch/docs',
await expect(directoryPreparer.prepare(mockEntity)).rejects.toThrow(
/Unable to resolve location type github/,
);
expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
});
});
@@ -15,104 +15,68 @@
*/
import { UrlReader } from '@backstage/backend-common';
import { InputError, NotModifiedError } from '@backstage/errors';
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import parseGitUrl from 'git-url-parse';
import path from 'path';
import { Logger } from 'winston';
import { InputError } from '@backstage/errors';
import {
checkoutGitRepository,
getLastCommitTimestamp,
parseReferenceAnnotation,
} from '../../helpers';
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import { Logger } from 'winston';
import { parseReferenceAnnotation, transformDirLocation } from '../../helpers';
import { PreparerBase, PreparerResponse } from './types';
export class DirectoryPreparer implements PreparerBase {
constructor(
private readonly config: Config,
private readonly logger: Logger,
private readonly reader: UrlReader,
) {
this.config = config;
this.logger = logger;
private readonly scmIntegrations: ScmIntegrationRegistry;
private readonly reader: UrlReader;
constructor(config: Config, _logger: Logger, reader: UrlReader) {
this.reader = reader;
this.scmIntegrations = ScmIntegrations.fromConfig(config);
}
private async resolveManagedByLocationToDir(
async prepare(
entity: Entity,
options?: { etag?: string },
options?: { logger?: Logger; etag?: string },
): Promise<PreparerResponse> {
const { type, target } = parseReferenceAnnotation(
'backstage.io/managed-by-location',
const annotation = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
this.logger.debug(
`Building docs for entity with type 'dir' and managed-by-location '${type}'`,
const { type, target } = transformDirLocation(
entity,
annotation,
this.scmIntegrations,
);
switch (type) {
case 'url': {
options?.logger?.debug(`Reading files from ${target}`);
// the target is an absolute url since it has already been transformed
const response = await this.reader.readTree(target, {
etag: options?.etag,
});
const preparedDir = await response.dir();
options?.logger?.debug(`Tree downloaded and stored at ${preparedDir}`);
return {
preparedDir,
etag: response.etag,
};
}
case 'github':
case 'gitlab':
case 'azure/api': {
const parsedGitLocation = parseGitUrl(target);
const repoLocation = await checkoutGitRepository(
target,
this.config,
this.logger,
);
// Check if etag has changed for cache invalidation.
const etag = await getLastCommitTimestamp(repoLocation, this.logger);
if (options?.etag === etag.toString()) {
throw new NotModifiedError();
}
case 'dir': {
return {
preparedDir: path.dirname(
path.join(repoLocation, parsedGitLocation.filepath),
),
etag: etag.toString(),
};
}
case 'file':
return {
preparedDir: path.dirname(target),
// the transformation already validated that the target is in a safe location
preparedDir: target,
// Instead of supporting caching on local sources, use techdocs-cli for local development and debugging.
etag: '',
};
}
default:
throw new InputError(`Unable to resolve location type ${type}`);
}
}
async prepare(entity: Entity): Promise<PreparerResponse> {
this.logger.warn(
'You are using the legacy dir preparer in TechDocs which will be removed in near future (March 2021). ' +
'Migrate to URL reader by updating `backstage.io/techdocs-ref` annotation in `catalog-info.yaml` ' +
'to be prefixed with `url:`. Read the migration guide and benefits at https://github.com/backstage/backstage/issues/4409 ',
);
const { target } = parseReferenceAnnotation(
'backstage.io/techdocs-ref',
entity,
);
// This will throw NotModified error if etag has not changed.
const response = await this.resolveManagedByLocationToDir(entity);
return {
preparedDir: path.resolve(response.preparedDir, target),
etag: response.etag,
};
}
}
+1
View File
@@ -35,6 +35,7 @@
"@backstage/catalog-model": "^0.9.0",
"@backstage/config": "^0.1.5",
"@backstage/errors": "^0.1.1",
"@backstage/integration": "^0.5.8",
"@backstage/search-common": "^0.1.2",
"@backstage/techdocs-common": "^0.6.8",
"@types/express": "^4.17.6",
@@ -20,6 +20,7 @@ import {
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotModifiedError } from '@backstage/errors';
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
GeneratorBase,
GeneratorBuilder,
@@ -43,6 +44,7 @@ type DocsBuilderArguments = {
entity: Entity;
logger: Logger;
config: Config;
scmIntegrations: ScmIntegrationRegistry;
logStream?: Writable;
};
@@ -53,6 +55,7 @@ export class DocsBuilder {
private entity: Entity;
private logger: Logger;
private config: Config;
private scmIntegrations: ScmIntegrationRegistry;
private logStream: Writable | undefined;
constructor({
@@ -62,6 +65,7 @@ export class DocsBuilder {
entity,
logger,
config,
scmIntegrations,
logStream,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
@@ -70,6 +74,7 @@ export class DocsBuilder {
this.entity = entity;
this.logger = logger;
this.config = config;
this.scmIntegrations = scmIntegrations;
this.logStream = logStream;
}
@@ -166,7 +171,10 @@ export class DocsBuilder {
path.join(tmpdirResolvedPath, 'techdocs-tmp-'),
);
const parsedLocationAnnotation = getLocationForEntity(this.entity);
const parsedLocationAnnotation = getLocationForEntity(
this.entity,
this.scmIntegrations,
);
await this.generator.run({
inputDir: preparedDir,
outputDir,
@@ -19,6 +19,7 @@ import {
PluginEndpointDiscovery,
} from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import {
GeneratorBuilder,
PreparerBuilder,
@@ -69,6 +70,7 @@ describe('DocsSynchronizer', () => {
publisher,
config: new ConfigReader({}),
logger: getVoidLogger(),
scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
});
});
@@ -17,6 +17,7 @@
import { Entity } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import { NotFoundError } from '@backstage/errors';
import { ScmIntegrationRegistry } from '@backstage/integration';
import {
GeneratorBuilder,
PreparerBuilder,
@@ -36,19 +37,23 @@ export class DocsSynchronizer {
private readonly publisher: PublisherBase;
private readonly logger: winston.Logger;
private readonly config: Config;
private readonly scmIntegrations: ScmIntegrationRegistry;
constructor({
publisher,
logger,
config,
scmIntegrations,
}: {
publisher: PublisherBase;
logger: winston.Logger;
config: Config;
scmIntegrations: ScmIntegrationRegistry;
}) {
this.config = config;
this.logger = logger;
this.publisher = publisher;
this.scmIntegrations = scmIntegrations;
}
async doSync({
@@ -87,19 +92,20 @@ export class DocsSynchronizer {
return;
}
const docsBuilder = new DocsBuilder({
preparers,
generators,
publisher: this.publisher,
logger: taskLogger,
entity,
config: this.config,
logStream,
});
let foundDocs = false;
try {
const docsBuilder = new DocsBuilder({
preparers,
generators,
publisher: this.publisher,
logger: taskLogger,
entity,
config: this.config,
scmIntegrations: this.scmIntegrations,
logStream,
});
const updated = await docsBuilder.build();
if (!updated) {
@@ -29,6 +29,7 @@ import express, { Response } from 'express';
import Router from 'express-promise-router';
import { Knex } from 'knex';
import { Logger } from 'winston';
import { ScmIntegrations } from '@backstage/integration';
import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
/**
@@ -79,10 +80,12 @@ export async function createRouter(
const router = Router();
const { publisher, config, logger, discovery } = options;
const catalogClient = new CatalogClient({ discoveryApi: discovery });
const scmIntegrations = ScmIntegrations.fromConfig(config);
const docsSynchronizer = new DocsSynchronizer({
publisher: publisher,
logger: logger,
config: config,
publisher,
logger,
config,
scmIntegrations,
});
router.get('/metadata/techdocs/:namespace/:kind/:name', async (req, res) => {
@@ -126,7 +129,7 @@ export async function createRouter(
)
).json()) as Entity;
const locationMetadata = getLocationForEntity(entity);
const locationMetadata = getLocationForEntity(entity, scmIntegrations);
res.json({ ...entity, locationMetadata });
} catch (err) {
logger.info(