diff --git a/docs/features/techdocs/getting-started.md b/docs/features/techdocs/getting-started.md index e0b323857d..efed896fd0 100644 --- a/docs/features/techdocs/getting-started.md +++ b/docs/features/techdocs/getting-started.md @@ -157,15 +157,17 @@ techdocs: builder: 'local' ``` -Set `techdocs.builder` to `'local'` if you want your TechDocs Backend to be -responsible for generating documentation sites. If set to `'external'`, -Backstage will assume that the sites are being generated on each entity's CI/CD -pipeline, and are being stored in a storage somewhere. +Note that we recommend generating docs on CI/CD instead. Read more in the +"Basic" and "Recommended" sections of the +[TechDocs Architecture](architecture.md). But if you want to get started quickly +set `techdocs.builder` to `'local'` so that TechDocs Backend is responsible for +generating documentation sites. If set to `'external'`, Backstage will assume +that the sites are being generated on each entity's CI/CD pipeline, and are +being stored in a storage somewhere. When `techdocs.builder` is set to `'external'`, TechDocs becomes more or less a read-only experience where it serves static files from a storage containing all -the generated documentation. Read more in the "Basic" and "Recommended" sections -of the [TechDocs Architecture](architecture.md). +the generated documentation. ### Choosing storage (publisher) diff --git a/packages/techdocs-common/src/stages/prepare/commonGit.test.ts b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts index 704cd23daa..b715b27407 100644 --- a/packages/techdocs-common/src/stages/prepare/commonGit.test.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts @@ -16,8 +16,8 @@ import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { CommonGitPreparer } from './commonGit'; import { checkoutGitRepository } from '../../helpers'; +import { CommonGitPreparer } from './commonGit'; function normalizePath(path: string) { return path @@ -57,9 +57,9 @@ describe('commonGit preparer', () => { 'github:https://github.com/backstage/backstage/blob/master/plugins/techdocs-backend/examples/documented-component', }); - const tempDocsPath = await preparer.prepare(mockEntity); + const { preparedDir } = await preparer.prepare(mockEntity); expect(checkoutGitRepository).toHaveBeenCalledTimes(1); - expect(normalizePath(tempDocsPath)).toEqual( + expect(normalizePath(preparedDir)).toEqual( '/tmp/backstage-repo/org/name/branch/plugins/techdocs-backend/examples/documented-component', ); }); @@ -72,9 +72,9 @@ describe('commonGit preparer', () => { 'gitlab:https://gitlab.com/xesjkeee/go-logger/blob/master/catalog-info.yaml', }); - const tempDocsPath = await preparer.prepare(mockEntity); + const { preparedDir } = await preparer.prepare(mockEntity); expect(checkoutGitRepository).toHaveBeenCalledTimes(2); - expect(normalizePath(tempDocsPath)).toEqual( + expect(normalizePath(preparedDir)).toEqual( '/tmp/backstage-repo/org/name/branch/catalog-info.yaml', ); }); @@ -87,9 +87,9 @@ describe('commonGit preparer', () => { 'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml', }); - const tempDocsPath = await preparer.prepare(mockEntity); + const { preparedDir } = await preparer.prepare(mockEntity); expect(checkoutGitRepository).toHaveBeenCalledTimes(3); - expect(normalizePath(tempDocsPath)).toEqual( + expect(normalizePath(preparedDir)).toEqual( '/tmp/backstage-repo/org/name/branch/template.yaml', ); }); diff --git a/packages/techdocs-common/src/stages/prepare/commonGit.ts b/packages/techdocs-common/src/stages/prepare/commonGit.ts index 7eac07d76f..0445da8187 100644 --- a/packages/techdocs-common/src/stages/prepare/commonGit.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.ts @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; -import parseGitUrl from 'git-url-parse'; import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { PreparerBase } from './types'; -import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; - +import parseGitUrl from 'git-url-parse'; +import path from 'path'; import { Logger } from 'winston'; +import { checkoutGitRepository, parseReferenceAnnotation } from '../../helpers'; +import { PreparerBase, PreparerResponse } from './types'; export class CommonGitPreparer implements PreparerBase { private readonly config: Config; @@ -31,7 +30,7 @@ export class CommonGitPreparer implements PreparerBase { this.logger = logger; } - async prepare(entity: Entity): Promise { + async prepare(entity: Entity): Promise { const { target } = parseReferenceAnnotation( 'backstage.io/techdocs-ref', entity, @@ -45,7 +44,13 @@ export class CommonGitPreparer implements PreparerBase { ); const parsedGitLocation = parseGitUrl(target); - return path.join(repoPath, parsedGitLocation.filepath); + // TODO: Return git commit sha + const etag = ''; + + return { + preparedDir: path.join(repoPath, parsedGitLocation.filepath), + etag, + }; } catch (error) { this.logger.debug(`Repo checkout failed with error ${error.message}`); throw error; diff --git a/packages/techdocs-common/src/stages/prepare/dir.test.ts b/packages/techdocs-common/src/stages/prepare/dir.test.ts index cd8d358afe..3538175622 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.test.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.test.ts @@ -15,8 +15,8 @@ */ import { getVoidLogger, UrlReader } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; -import { DirectoryPreparer } from './dir'; import { checkoutGitRepository } from '../../helpers'; +import { DirectoryPreparer } from './dir'; function normalizePath(path: string) { return path @@ -66,9 +66,8 @@ describe('directory preparer', () => { 'backstage.io/techdocs-ref': 'dir:./our-documentation', }); - expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual( - '/directory/our-documentation', - ); + const { preparedDir } = await directoryPreparer.prepare(mockEntity); + expect(normalizePath(preparedDir)).toEqual('/directory/our-documentation'); }); it('should merge managed-by-location and techdocs-ref when techdocs-ref is absolute', async () => { @@ -84,9 +83,8 @@ describe('directory preparer', () => { 'backstage.io/techdocs-ref': 'dir:/our-documentation/techdocs', }); - expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual( - '/our-documentation/techdocs', - ); + const { preparedDir } = await directoryPreparer.prepare(mockEntity); + expect(normalizePath(preparedDir)).toEqual('/our-documentation/techdocs'); }); it('should merge managed-by-location and techdocs-ref when managed-by-location is a git repository', async () => { @@ -102,7 +100,8 @@ describe('directory preparer', () => { 'backstage.io/techdocs-ref': 'dir:./docs', }); - expect(normalizePath(await directoryPreparer.prepare(mockEntity))).toEqual( + const { preparedDir } = await directoryPreparer.prepare(mockEntity); + expect(normalizePath(preparedDir)).toEqual( '/tmp/backstage-repo/org/name/branch/docs', ); expect(checkoutGitRepository).toHaveBeenCalledTimes(1); diff --git a/packages/techdocs-common/src/stages/prepare/dir.ts b/packages/techdocs-common/src/stages/prepare/dir.ts index ab277fda9a..e59a3f02b6 100644 --- a/packages/techdocs-common/src/stages/prepare/dir.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.ts @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { PreparerBase } from './types'; +import { InputError, UrlReader } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import path from 'path'; -import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; -import { UrlReader, InputError } from '@backstage/backend-common'; import parseGitUrl from 'git-url-parse'; +import path from 'path'; import { Logger } from 'winston'; +import { checkoutGitRepository, parseReferenceAnnotation } from '../../helpers'; +import { PreparerBase, PreparerResponse } from './types'; export class DirectoryPreparer implements PreparerBase { constructor( @@ -68,7 +68,7 @@ export class DirectoryPreparer implements PreparerBase { } } - async prepare(entity: Entity): Promise { + async prepare(entity: Entity): Promise { const { target } = parseReferenceAnnotation( 'backstage.io/techdocs-ref', entity, @@ -78,8 +78,12 @@ export class DirectoryPreparer implements PreparerBase { entity, ); - return new Promise(resolve => { - resolve(path.resolve(managedByLocationDirectory, target)); - }); + // TODO: etag will be returned as a commit sha from resolveManagedByLocationToDir. + const etag = ''; + + return { + preparedDir: path.resolve(managedByLocationDirectory, target), + etag, + }; } } diff --git a/packages/techdocs-common/src/stages/prepare/types.ts b/packages/techdocs-common/src/stages/prepare/types.ts index 97e75306c3..c8ea9c611a 100644 --- a/packages/techdocs-common/src/stages/prepare/types.ts +++ b/packages/techdocs-common/src/stages/prepare/types.ts @@ -16,13 +16,31 @@ import type { Entity } from '@backstage/catalog-model'; import { Logger } from 'winston'; +export type PreparerResponse = { + /** + * The path to directory where the tree is downloaded. + */ + preparedDir: string; + /** + * A unique identifer of the tree blob, usually the commit SHA or etag from the target. + */ + etag: string; +}; + export type PreparerBase = { /** * Given an Entity definition from the Service Catalog, go and prepare a directory - * with contents from the location in temporary storage and return the path + * with contents from the location in temporary storage and return the path. + * * @param entity The entity from the Service Catalog + * @param opts.etag (Optional) If etag is provider, it will be used to check if the target has + * updated since the last build. + * @throws {NotModifiedError} when the prepared directory has not been changed since the last build. */ - prepare(entity: Entity, opts?: { logger: Logger }): Promise; + prepare( + entity: Entity, + opts?: { logger: Logger; etag?: string }, + ): Promise; }; export type PreparerBuilder = { @@ -30,10 +48,14 @@ export type PreparerBuilder = { get(entity: Entity): PreparerBase; }; +/** + * Everything except `url` will be deprecated. + * Read more https://github.com/backstage/backstage/issues/4409 + */ export type RemoteProtocol = + | 'url' | 'dir' | 'github' | 'gitlab' | 'file' - | 'azure/api' - | 'url'; + | 'azure/api'; diff --git a/packages/techdocs-common/src/stages/prepare/url.ts b/packages/techdocs-common/src/stages/prepare/url.ts index b938c6b5bc..c10822dace 100644 --- a/packages/techdocs-common/src/stages/prepare/url.ts +++ b/packages/techdocs-common/src/stages/prepare/url.ts @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; import { UrlReader } from '@backstage/backend-common'; -import { PreparerBase } from './types'; +import { Entity } from '@backstage/catalog-model'; +import { Logger } from 'winston'; import { getDocFilesFromRepository } from '../../helpers'; +import { PreparerBase, PreparerResponse } from './types'; export class UrlPreparer implements PreparerBase { private readonly logger: Logger; @@ -28,9 +28,15 @@ export class UrlPreparer implements PreparerBase { this.reader = reader; } - async prepare(entity: Entity): Promise { + async prepare(entity: Entity): Promise { try { - return getDocFilesFromRepository(this.reader, entity); + const preparedDir = await getDocFilesFromRepository(this.reader, entity); + // TODO: This will be actual etag from URL Reader. + const etag = ''; + return { + preparedDir, + etag, + }; } catch (error) { this.logger.debug( `Unable to fetch files for building docs ${error.message}`, diff --git a/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts b/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts index d6a19764d9..eecb161074 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts @@ -18,6 +18,11 @@ type buildInfo = { [key: string]: number; }; +// TODO: Build info should be part of TechDocs storage, inside `techdocs_metadata.json` +// instead of in-memory storage of the Backstage instance. +// In case of multi-region Backstage deployments, or even using multiple Kubernetes pods, +// if each instance creates its separate build info in-memory, it will result in duplicate +// builds per instance. Also if the pod restarts, all the sites will have to be re-built. const builds = {} as buildInfo; /** diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts index 33d3120c47..7a3cbef1e9 100644 --- a/plugins/techdocs-backend/src/DocsBuilder/builder.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -13,22 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -import Docker from 'dockerode'; -import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { + GeneratorBase, + GeneratorBuilder, + getLastCommitTimestamp, + getLocationForEntity, + PreparerBase, PreparerBuilder, PublisherBase, - GeneratorBuilder, - PreparerBase, - GeneratorBase, - getLocationForEntity, - getLastCommitTimestamp, } from '@backstage/techdocs-common'; +import Docker from 'dockerode'; +import fs from 'fs-extra'; +import os from 'os'; +import path from 'path'; +import { Logger } from 'winston'; import { BuildMetadataStorage } from '.'; const getEntityId = (entity: Entity) => { @@ -76,7 +76,9 @@ export class DocsBuilder { public async build() { this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`); - const preparedDir = await this.preparer.prepare(this.entity); + // TODO: This will throw an error if no further build is required, site is cached. + // TODO: Use etag from here and store in memory. + const { preparedDir } = await this.preparer.prepare(this.entity); const parsedLocationAnnotation = getLocationForEntity(this.entity); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index fe7b52ab52..a379e47bfb 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -13,23 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Logger } from 'winston'; -import Router from 'express-promise-router'; -import express from 'express'; -import Knex from 'knex'; -import fetch from 'cross-fetch'; -import { Config } from '@backstage/config'; -import Docker from 'dockerode'; -import { - GeneratorBuilder, - PreparerBuilder, - PublisherBase, - getLocationForEntity, -} from '@backstage/techdocs-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { getEntityNameFromUrlPath } from './helpers'; +import { Config } from '@backstage/config'; +import { + GeneratorBuilder, + getLocationForEntity, + PreparerBuilder, + PublisherBase, +} from '@backstage/techdocs-common'; +import fetch from 'cross-fetch'; +import Docker from 'dockerode'; +import express from 'express'; +import Router from 'express-promise-router'; +import Knex from 'knex'; +import { Logger } from 'winston'; import { DocsBuilder } from '../DocsBuilder'; +import { getEntityNameFromUrlPath } from './helpers'; type RouterOptions = { preparers: PreparerBuilder; @@ -145,9 +145,7 @@ export async function createRouter({ }); switch (publisherType) { case 'local': - if (!(await docsBuilder.docsUpToDate())) { - await docsBuilder.build(); - } + await docsBuilder.build(); break; case 'awsS3': case 'azureBlobStorage': @@ -186,9 +184,11 @@ export async function createRouter({ 'Found pre-generated docs for this entity. Serving them.', ); // TODO: re-trigger build for cache invalidation. - // Compare the date modified of the requested file on storage and compare it against - // the last modified or last commit timestamp in the repository. + // Add build info in techdocs_metadata.json and compare it against + // the eTag/commit in the repository. // Without this, docs will not be re-built once they have been generated. + // Although it is unconventional that anyone will face this issue - because + // if you have an external storage, you should be using CI/CD to build and publish docs. } break; default: