Merge pull request #10378 from bridgetbarnes/techdocs-sync-log

[TechDocs] Use DocsSyncroniser logger to get doc build logs

.changeset/techdocs-paws-study.md

@@ -0,0 +1,64 @@
+---
+'@backstage/plugin-techdocs-backend': patch
+---
+
+Output logs from a TechDocs build to a logging transport in addition to existing
+frontend event stream, for capturing these logs to other sources.
+
+This allows users to capture debugging information around why tech docs fail to build
+without needing to rely on end users capturing information from their web browser.
+
+The most common use case is to log to the same place as the rest of the backend
+application logs.
+
+Sample usage:
+
+```
+import { DockerContainerRunner } from '@backstage/backend-common';
+import {
+  createRouter,
+  Generators,
+  Preparers,
+  Publisher,
+} from '@backstage/plugin-techdocs-backend';
+import Docker from 'dockerode';
+import { Router } from 'express';
+import { PluginEnvironment } from '../types';
+
+export default async function createPlugin(
+  env: PluginEnvironment,
+): Promise<Router> {
+  const preparers = await Preparers.fromConfig(env.config, {
+    logger: env.logger,
+    reader: env.reader,
+  });
+
+  const dockerClient = new Docker();
+  const containerRunner = new DockerContainerRunner({ dockerClient });
+
+  const generators = await Generators.fromConfig(env.config, {
+    logger: env.logger,
+    containerRunner,
+  });
+
+  const publisher = await Publisher.fromConfig(env.config, {
+    logger: env.logger,
+    discovery: env.discovery,
+  });
+
+  await publisher.getReadiness();
+
+  return await createRouter({
+    preparers,
+    generators,
+    publisher,
+    logger: env.logger,
+    // Passing a buildLogTransport as a parameter in createRouter will enable
+    // capturing build logs to a backend log stream
+    buildLogTransport: env.logger,
+    config: env.config,
+    discovery: env.discovery,
+    cache: env.cache,
+  });
+}
+```

plugins/techdocs-backend/api-report.md

@@ -21,6 +21,7 @@ import { PublisherBase } from '@backstage/plugin-techdocs-node';
 import { Readable } from 'stream';
 import { TechDocsDocument } from '@backstage/plugin-techdocs-node';
 import { TokenManager } from '@backstage/backend-common';
+import * as winston from 'winston';
 
 // @public
 export function createRouter(options: RouterOptions): Promise<express.Router>;
@@ -71,22 +72,24 @@ export type OutOfTheBoxDeploymentOptions = {
   preparers: PreparerBuilder;
   generators: GeneratorBuilder;
   publisher: PublisherBase;
-  logger: Logger;
+  logger: winston.Logger;
   discovery: PluginEndpointDiscovery;
   database?: Knex;
   config: Config;
   cache: PluginCacheManager;
   docsBuildStrategy?: DocsBuildStrategy;
+  buildLogTransport?: winston.transport;
 };
 
 // @public
 export type RecommendedDeploymentOptions = {
   publisher: PublisherBase;
-  logger: Logger;
+  logger: winston.Logger;
   discovery: PluginEndpointDiscovery;
   config: Config;
   cache: PluginCacheManager;
   docsBuildStrategy?: DocsBuildStrategy;
+  buildLogTransport?: winston.transport;
 };
 
 // @public

plugins/techdocs-backend/src/service/DocsSynchronizer.test.ts

@@ -25,6 +25,8 @@ import {
   PreparerBuilder,
   PublisherBase,
 } from '@backstage/plugin-techdocs-node';
+import { PassThrough } from 'stream';
+import * as winston from 'winston';
 import { TechDocsCache } from '../cache';
 import { DocsBuilder, shouldCheckForUpdate } from '../DocsBuilder';
 import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
@@ -74,12 +76,17 @@ describe('DocsSynchronizer', () => {
   } as unknown as jest.Mocked<TechDocsCache>;
 
   let docsSynchronizer: DocsSynchronizer;
+
   const mockResponseHandler: jest.Mocked<DocsSynchronizerSyncOpts> = {
     log: jest.fn(),
     finish: jest.fn(),
     error: jest.fn(),
   };
 
+  const mockBuildLogTransport = new winston.transports.Stream({
+    stream: new PassThrough(),
+  });
+
   beforeEach(async () => {
     publisher.docsRouter.mockReturnValue(() => {});
     discovery.getBaseUrl.mockImplementation(async type => {
@@ -90,6 +97,7 @@ describe('DocsSynchronizer', () => {
       publisher,
       config: new ConfigReader({}),
       logger: getVoidLogger(),
+      buildLogTransport: mockBuildLogTransport,
       scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
       cache,
     });
@@ -289,6 +297,9 @@ describe('DocsSynchronizer', () => {
           techdocs: { legacyUseCaseSensitiveTripletPaths: true },
         }),
         logger: getVoidLogger(),
+        buildLogTransport: new winston.transports.Stream({
+          stream: new PassThrough(),
+        }),
         scmIntegrations: ScmIntegrations.fromConfig(new ConfigReader({})),
         cache,
       });
@@ -321,5 +332,22 @@ describe('DocsSynchronizer', () => {
 
       expect(mockResponseHandler.finish).toBeCalledWith({ updated: false });
     });
+
+    it("adds the build log transport to the logger's list of transports", async () => {
+      let logger: winston.Logger;
+      MockedDocsBuilder.prototype.build.mockImplementation(async () => {
+        logger = MockedDocsBuilder.mock.calls[0][0].logger;
+        expect(logger.transports).toContain(mockBuildLogTransport);
+
+        return true;
+      });
+
+      await docsSynchronizer.doSync({
+        responseHandler: mockResponseHandler,
+        entity,
+        preparers,
+        generators,
+      });
+    });
   });
 });

plugins/techdocs-backend/src/service/DocsSynchronizer.ts

@@ -43,6 +43,7 @@ export type DocsSynchronizerSyncOpts = {
 export class DocsSynchronizer {
   private readonly publisher: PublisherBase;
   private readonly logger: winston.Logger;
+  private readonly buildLogTransport: winston.transport;
   private readonly config: Config;
   private readonly scmIntegrations: ScmIntegrationRegistry;
   private readonly cache: TechDocsCache | undefined;
@@ -50,18 +51,21 @@ export class DocsSynchronizer {
   constructor({
     publisher,
     logger,
+    buildLogTransport,
     config,
     scmIntegrations,
     cache,
   }: {
     publisher: PublisherBase;
     logger: winston.Logger;
+    buildLogTransport: winston.transport;
     config: Config;
     scmIntegrations: ScmIntegrationRegistry;
     cache: TechDocsCache | undefined;
   }) {
     this.config = config;
     this.logger = logger;
+    this.buildLogTransport = buildLogTransport;
     this.publisher = publisher;
     this.scmIntegrations = scmIntegrations;
     this.cache = cache;
@@ -96,6 +100,7 @@ export class DocsSynchronizer {
     });
 
     taskLogger.add(new winston.transports.Stream({ stream: logStream }));
+    taskLogger.add(this.buildLogTransport);
 
     // check if the last update check was too recent
     if (!shouldCheckForUpdate(entity.metadata.uid!)) {

plugins/techdocs-backend/src/service/router.ts

@@ -30,7 +30,6 @@ import {
 import express, { Response } from 'express';
 import Router from 'express-promise-router';
 import { Knex } from 'knex';
-import { Logger } from 'winston';
 import { ScmIntegrations } from '@backstage/integration';
 import { DocsSynchronizer, DocsSynchronizerSyncOpts } from './DocsSynchronizer';
 import { createCacheMiddleware, TechDocsCache } from '../cache';
@@ -39,6 +38,8 @@ import {
   DefaultDocsBuildStrategy,
   DocsBuildStrategy,
 } from './DocsBuildStrategy';
+import * as winston from 'winston';
+import { PassThrough } from 'stream';
 
 /**
  * Required dependencies for running TechDocs in the "out-of-the-box"
@@ -50,12 +51,13 @@ export type OutOfTheBoxDeploymentOptions = {
   preparers: PreparerBuilder;
   generators: GeneratorBuilder;
   publisher: PublisherBase;
-  logger: Logger;
+  logger: winston.Logger;
   discovery: PluginEndpointDiscovery;
   database?: Knex; // TODO: Make database required when we're implementing database stuff.
   config: Config;
   cache: PluginCacheManager;
   docsBuildStrategy?: DocsBuildStrategy;
+  buildLogTransport?: winston.transport;
 };
 
 /**
@@ -66,11 +68,12 @@ export type OutOfTheBoxDeploymentOptions = {
  */
 export type RecommendedDeploymentOptions = {
   publisher: PublisherBase;
-  logger: Logger;
+  logger: winston.Logger;
   discovery: PluginEndpointDiscovery;
   config: Config;
   cache: PluginCacheManager;
   docsBuildStrategy?: DocsBuildStrategy;
+  buildLogTransport?: winston.transport;
 };
 
 /**
@@ -107,6 +110,9 @@ export async function createRouter(
   const catalogClient = new CatalogClient({ discoveryApi: discovery });
   const docsBuildStrategy =
     options.docsBuildStrategy ?? DefaultDocsBuildStrategy.fromConfig(config);
+  const buildLogTransport =
+    options.buildLogTransport ??
+    new winston.transports.Stream({ stream: new PassThrough() });
 
   // Entities are cached to optimize the /static/docs request path, which can be called many times
   // when loading a single techdocs page.
@@ -127,6 +133,7 @@ export async function createRouter(
   const docsSynchronizer = new DocsSynchronizer({
     publisher,
     logger,
+    buildLogTransport,
     config,
     scmIntegrations,
     cache,