diff --git a/packages/docgen/.eslintrc.js b/packages/docgen/.eslintrc.js new file mode 100644 index 0000000000..503c048748 --- /dev/null +++ b/packages/docgen/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], + rules: { + 'no-console': 0, + }, +}; diff --git a/packages/docgen/src/compiler.ts b/packages/docgen/src/compiler.ts new file mode 100644 index 0000000000..3a196ca709 --- /dev/null +++ b/packages/docgen/src/compiler.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as ts from 'typescript'; +import { resolve, join, dirname } from 'path'; +import { promisify } from 'util'; +import fs from 'fs'; +import ApiDocGenerator from './docgen/ApiDocGenerator'; +import sortSelector from './docgen/sortSelector'; +import TypeLocator from './docgen/TypeLocator'; +import ApiDocPrinter from './docgen/ApiDocPrinter'; +import TypescriptHighlighter from './docgen/TypescriptHighlighter'; +import MarkdownPrinter from './docgen/MarkdownPrinter'; +import { sync as mkdirpSync } from 'mkdirp'; + +const writeFile = promisify(fs.writeFile); + +function loadOptions(path: string): ts.CompilerOptions { + let { extends: parent, compilerOptions }: any = require(path); + + if (!parent) { + return compilerOptions; + } + if (parent.startsWith('.')) { + parent = join(dirname(path), parent); + } + + return { ...loadOptions(parent), ...compilerOptions }; +} + +async function main() { + const rootDir = resolve(__dirname, '..'); + const srcDir = resolve(rootDir, 'src'); + const entrypoint = resolve(srcDir, 'index.js'); + const apiRefsDir = resolve(rootDir, 'docgen', 'build', 'api-refs'); + const mkdocsYaml = resolve(apiRefsDir, 'mkdocs.yml'); + + process.chdir(rootDir); + + const options = loadOptions('../tsconfig.json'); + + delete options.moduleResolution; + options.removeComments = false; + options.noEmit = true; + + const program = ts.createProgram([entrypoint], options); + + const typeLocator = TypeLocator.fromProgram(program); + + const { apis } = typeLocator.findExportedInstances({ + apis: typeLocator.getExportedType( + resolve(srcDir, 'core', 'api', 'ApiRef.ts'), + ), + }); + + const apiDocGenerator = ApiDocGenerator.fromProgram(program, rootDir, srcDir); + + const apiDocs = apis + .map(api => { + try { + return apiDocGenerator.toDoc(api); + } catch (error) { + throw new Error( + `Doc generation failed for API in ${api.source.fileName}, ${error.stack}`, + ); + } + }) + .sort(sortSelector(x => x.id)); + + const apiDocPrinter = new ApiDocPrinter( + () => new MarkdownPrinter(new TypescriptHighlighter()), + ); + + mkdirpSync(resolve(apiRefsDir, 'docs')); + + await Promise.all( + apiDocs.map(apiDoc => { + const data = apiDocPrinter.print(apiDoc); + + return writeFile(join(apiRefsDir, 'docs', `${apiDoc.name}.md`), data); + }), + ); + + fs.writeFileSync( + mkdocsYaml, + [ + 'site_name: api-references', + 'nav:', + ...apiDocs.map(({ id, name }) => ` - ${id}: '${name}.md'`), + ].join('\n'), + 'utf8', + ); +} + +main().catch(error => { + console.error(error.stack || error); + process.exit(1); +}); diff --git a/packages/docgen/src/docgen/ApiDocGenerator.test.ts b/packages/docgen/src/docgen/ApiDocGenerator.test.ts new file mode 100644 index 0000000000..f30c67288d --- /dev/null +++ b/packages/docgen/src/docgen/ApiDocGenerator.test.ts @@ -0,0 +1,261 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; +import TypeLocator from './TypeLocator'; +import { createMemProgram } from './testUtils'; +import ApiDocGenerator from './ApiDocGenerator'; + +describe('ApiDocGenerator', () => { + it('should generate empty API doc', () => { + const program = createMemProgram( + ` + import MyApi from './type'; + + type MyApiType = {}; + + export const myApi = new MyApi({ + id: 'my-id', + description: 'my-description', + }); + `, + { + '/mem/type.ts': `export default class MyApi { + constructor(private readonly info: { id: string, description: string }) {} + }`, + }, + ); + + const typeLocator = TypeLocator.fromProgram(program); + + const { apiInstances } = typeLocator.findExportedInstances({ + apiInstances: typeLocator.getExportedType('/mem/type.ts'), + }); + + expect(apiInstances.length).toBe(1); + const [apiInstance] = apiInstances; + + const docGenerator = ApiDocGenerator.fromProgram(program, '/mem', '/mem'); + const doc = docGenerator.toDoc(apiInstance); + + expect(doc.id).toBe('my-id'); + expect(doc.description).toBe('my-description'); + expect(doc.name).toBe('myApi'); + expect(doc.source.fileName).toBe('/mem/index.ts'); + expect(doc.interfaceInfo.dependentTypes).toEqual([]); + expect(doc.interfaceInfo.docs).toEqual([]); + expect(doc.interfaceInfo.file).toBe('index.ts'); + expect(doc.interfaceInfo.lineInFile).toBe(4); + expect(doc.interfaceInfo.members).toEqual([]); + expect(doc.interfaceInfo.name).toBe('MyApiType'); + }); + + it('should generate API docs', () => { + const program = createMemProgram( + ` + import MyApi from './type'; + + /** MySubSubType Docs */ + type MySubSubType = { n: number; }; + + /** MySubType Docs */ + type MySubType = { + /** Field a docs */ + a: boolean; + // Field b docs + b: MySubSubType; + } + + /** MySecondSubType Docs */ + + /** With multiple comments */ + export type MySecondSubType = { s: string }; + + // MyThirdSubType Docs that shouldn't show up + type MyThirdSubType = { b: boolean }; + + /** MyApiType Docs */ + type MyApiType = { + /** Docs for x */ + x: string; + // Line comments shouldn't show up + y: MySubType; + /** Multiple */ + /** JsDoc */ + /** Comments */ + z(a: Promise): Array; + }; + + /** Should not show up */ + export const myApi = new MyApi({ + id: 'my-id', + description: 'my-description', + }); + `, + { + '/mem/type.ts': `export default class MyApi { + constructor(private readonly info: { id: string, description: string }) {} + }`, + }, + ); + + const source = program.getSourceFile('/mem/index.ts'); + + // Figure out type IDs so we can make sure they match later + const checker = program.getTypeChecker(); + const symbols = checker.getSymbolsInScope( + source!.getChildren().slice(-1)[0], + ts.SymbolFlags.TypeAlias, + ); + const Ids = [ + 'MySubType', + 'MySubSubType', + 'MySecondSubType', + 'MyThirdSubType', + ].reduce((ids, name) => { + const symbol = symbols.find(s => s.escapedName === name)!; + const type = checker.getTypeAtLocation(symbol.declarations[0]); + ids[name] = (type.aliasSymbol as any).id; + return ids; + }, {} as { [key in string]: number }); + + const typeLocator = TypeLocator.fromProgram(program); + + const { apiInstances } = typeLocator.findExportedInstances({ + apiInstances: typeLocator.getExportedType('/mem/type.ts'), + }); + + expect(apiInstances.length).toBe(1); + const [apiInstance] = apiInstances; + + const docGenerator = ApiDocGenerator.fromProgram(program, '/mem', '/mem'); + const doc = docGenerator.toDoc(apiInstance); + + expect(doc.id).toBe('my-id'); + expect(doc.description).toBe('my-description'); + expect(doc.name).toBe('myApi'); + expect(doc.source.fileName).toBe('/mem/index.ts'); + expect(doc.interfaceInfo.docs).toEqual(['MyApiType Docs']); + expect(doc.interfaceInfo.file).toBe('index.ts'); + expect(doc.interfaceInfo.lineInFile).toBe(24); + expect(doc.interfaceInfo.name).toBe('MyApiType'); + expect(doc.interfaceInfo.members).toEqual([ + { + type: 'prop', + name: 'x', + path: 'MyApiType.x', + text: 'x: string', + docs: ['Docs for x'], + links: [], + }, + { + type: 'prop', + name: 'y', + path: 'MyApiType.y', + text: 'y: MySubType', + docs: [], + links: [ + { + id: Ids.MySubType, + name: 'MySubType', + path: 'index.ts/MySubType', + location: [3, 12], + }, + ], + }, + { + type: 'method', + name: 'z', + path: 'MyApiType.z', + text: + 'z(a: Promise): Array', + docs: ['Multiple', 'JsDoc', 'Comments'], + links: [ + { + id: Ids.MySecondSubType, + name: 'MySecondSubType', + path: 'index.ts/MySecondSubType', + location: [27, 42], + }, + { + id: Ids.MyThirdSubType, + name: 'MyThirdSubType', + path: 'index.ts/MyThirdSubType', + location: [56, 70], + }, + ], + }, + ]); + expect(doc.interfaceInfo.dependentTypes).toEqual([ + { + id: Ids.MySubType, + name: 'MySubType', + path: 'index.ts/MySubType', + file: 'index.ts', + lineInFile: 8, + text: `type MySubType = { + /** Field a docs */ + a: boolean; + // Field b docs + b: MySubSubType; + }`, + docs: ['MySubType Docs'], + links: [ + { + id: Ids.MySubSubType, + name: 'MySubSubType', + path: 'index.ts/MySubSubType', + location: [102, 114], + }, + ], + children: [], + }, + { + id: Ids.MySubSubType, + name: 'MySubSubType', + path: 'index.ts/MySubSubType', + file: 'index.ts', + lineInFile: 5, + text: 'type MySubSubType = { n: number; }', + docs: ['MySubSubType Docs'], + links: [], + children: [], + }, + { + id: Ids.MySecondSubType, + name: 'MySecondSubType', + path: 'index.ts/MySecondSubType', + file: 'index.ts', + lineInFile: 18, + text: 'export type MySecondSubType = { s: string }', + docs: ['MySecondSubType Docs', 'With multiple comments'], + links: [], + children: [], + }, + { + id: Ids.MyThirdSubType, + name: 'MyThirdSubType', + path: 'index.ts/MyThirdSubType', + file: 'index.ts', + lineInFile: 21, + text: 'type MyThirdSubType = { b: boolean }', + docs: [], + links: [], + children: [], + }, + ]); + }); +}); diff --git a/packages/docgen/src/docgen/ApiDocGenerator.ts b/packages/docgen/src/docgen/ApiDocGenerator.ts new file mode 100644 index 0000000000..e1d1e0f5f1 --- /dev/null +++ b/packages/docgen/src/docgen/ApiDocGenerator.ts @@ -0,0 +1,297 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; +import { relative } from 'path'; +import { + ExportedInstance, + ApiDoc, + InterfaceInfo, + FieldInfo, + TypeInfo, + TypeLink, +} from './types'; +import { flatMap } from './flatMap'; + +/** + * The ApiDocGenerator uses the typescript compiler API to build the data structure that + * describes a Backstage API and all of it's related types. + * + * It receives an exported instance that of the form + * `export name = new Api({id: ..., description: ...})`. + * It will then traverse all fields and methods on the interface type, and also + * types and declaration of all types that are used by those, both directly and indirectly. + * While traversing, it collects information such as names, location in source, docs, etc. + */ +export default class ApiDocGenerator { + static fromProgram( + program: ts.Program, + basePath: string, + sourcePath: string, + ) { + return new ApiDocGenerator(program.getTypeChecker(), basePath, sourcePath); + } + + constructor( + private readonly checker: ts.TypeChecker, + private readonly basePath: string, + private readonly sourcePath: string, + ) {} + + toDoc(apiInstance: ExportedInstance): ApiDoc { + const { name, source, args, typeArgs } = apiInstance; + + const [info] = args; + if (!ts.isObjectLiteralExpression(info)) { + throw new Error('api info is not an object literal'); + } + + const id = this.getObjectPropertyLiteral(info, 'id'); + const description = this.getObjectPropertyLiteral(info, 'description'); + const interfaceInfo = this.getInterfaceInfo(typeArgs[0]); + if (!interfaceInfo) { + throw new Error('api has no info type argument'); + } + + return { id, name, source, description, interfaceInfo }; + } + + private getNodeDocs(node: ts.Node): string[] { + const docNodes = ((node && (node as any).jsDoc) || []) as ts.JSDoc[]; + const docs = docNodes.map(docNode => docNode.comment || '').filter(Boolean); + return docs; + } + + private getInterfaceInfo(typeNode: ts.TypeNode): InterfaceInfo { + if (ts.isTypeQueryNode(typeNode)) { + throw new Error( + 'APIs must have a proper type parameter, TypeQueries are now allowed', + ); + } + if (!ts.isTypeReferenceNode(typeNode)) { + throw new Error('Interface is not a type node'); + } + + const type = this.checker.getTypeFromTypeNode(typeNode); + const interfaceMembers = type.symbol.members as Map; + if (!interfaceMembers) { + throw new Error('Interface does not have any members'); + } + const name = (type.aliasSymbol || type.symbol).name; + const [declaration] = (type.aliasSymbol || type.symbol).declarations; + const sourceFile = declaration.getSourceFile(); + const file = relative(this.basePath, sourceFile.fileName); + const { line } = sourceFile.getLineAndCharacterOfPosition( + declaration.getStart(), + ); + const docs = this.getNodeDocs(declaration); + + const membersAndTypes = flatMap( + Array.from(interfaceMembers.values()), + fieldSymbol => this.getMemberInfo(name, fieldSymbol), + ); + + const members = membersAndTypes.map(t => t.member); + const dependentTypes = this.flattenTypes( + flatMap(membersAndTypes, t => t.dependentTypes), + ); + + return { name, docs, file, lineInFile: line + 1, members, dependentTypes }; + } + + private getObjectPropertyLiteral( + objectLiteral: ts.ObjectLiteralExpression, + propertyName: string, + ): string { + const prop = objectLiteral.properties.filter( + prop => + ts.isPropertyAssignment(prop) && + ts.isIdentifier(prop.name) && + prop.name.text === propertyName, + )[0] as ts.PropertyAssignment; + + if (!prop) { + throw new Error(`no identifier found for property ${propertyName}`); + } + + const { initializer } = prop; + if (!ts.isStringLiteral(initializer)) { + throw new Error(`no string literal for ${propertyName}`); + } + + return initializer.text; + } + + private getMemberInfo( + parentName: string, + symbol: ts.Symbol, + ): { member: FieldInfo; dependentTypes: TypeInfo[] } { + const declaration = symbol.valueDeclaration; + + const { links, infos: dependentTypes } = this.findAllTypeReferences( + declaration, + ); + + let type: FieldInfo['type'] = 'prop'; + if ( + ts.isPropertySignature(declaration) && + declaration.type && + ts.isFunctionTypeNode(declaration.type) + ) { + type = 'method'; + } else if (ts.isMethodSignature(declaration)) { + type = 'method'; + } + + return { + member: { + type, + name: symbol.name, + path: `${parentName}.${symbol.name}`, + text: declaration.getText().replace(/;$/, ''), + docs: this.getNodeDocs(declaration), + links: this.recalculateLinkOffsets(declaration, links), + }, + dependentTypes, + }; + } + + private findAllTypeReferences = ( + node: ts.Node | undefined, + visited: Set = new Set(), + ): { links: TypeLink[]; infos: TypeInfo[] } => { + if (!node || visited.has(node)) { + return { links: [], infos: [] }; + } + // This makes sure we don't end up in a loop. + // It doesn't exclude repeated types, e.g. Array>, since we're exiting on duplicate nodes, not types. + visited.add(node); + + const children = flatMap(node.getChildren(), child => + this.findAllTypeReferences(child, visited), + ); + + if (!ts.isTypeReferenceNode(node)) { + return { + links: flatMap(children, child => child.links), + infos: flatMap(children, child => child.infos), + }; + } + + const type = this.checker.getTypeFromTypeNode(node); + + const info = this.validateTypeDeclaration(type); + if (!info) { + return { + links: flatMap(children, child => child.links), + infos: flatMap(children, child => child.infos), + }; + } + const { symbol, declaration } = info; + + const typeRefs = this.findAllTypeReferences(declaration, visited); + + const sourceFile = declaration.getSourceFile(); + const { line } = sourceFile.getLineAndCharacterOfPosition( + declaration.getStart(), + ); + const file = relative(this.basePath, sourceFile.fileName); + const typeInfo = { + id: (symbol as any).id, + name: symbol.name, + text: declaration.getText().replace(/;$/, ''), + docs: this.getNodeDocs(declaration), + path: `${file}/${symbol.name}`, + file, + lineInFile: line + 1, // TS line is 0-based + links: this.recalculateLinkOffsets(declaration, typeRefs.links), + children: typeRefs.infos, + }; + + const link = { + id: typeInfo.id, + path: typeInfo.path, + name: typeInfo.name, + location: [node.getStart(), node.getEnd()] as const, + }; + + return { + links: [link, ...flatMap(children, child => child.links)], + infos: [typeInfo, ...flatMap(children, child => child.infos)], + }; + }; + + private validateTypeDeclaration( + type: ts.Type | undefined, + ): undefined | { symbol: ts.Symbol; declaration: ts.Declaration } { + if (!type) { + return; + } + const symbol = type.aliasSymbol || type.symbol; + if (!symbol) { + return; + } + + const [declaration] = symbol.declarations; + + // Don't generate standalone infos for type paramters + if (ts.isTypeParameterDeclaration(declaration)) { + return; + } + + if (!declaration.getSourceFile().fileName.startsWith(this.sourcePath)) { + return; + } + + return { symbol, declaration }; + } + + private flattenTypes(descs: TypeInfo[]): TypeInfo[] { + function getChildren(parent: TypeInfo): TypeInfo[] { + return [ + { + ...parent, + children: [], + }, + ...flatMap(parent.children, getChildren), + ]; + } + + const flatDescs = flatMap(descs, getChildren); + const seenTypes = new Set(); + return flatDescs.filter(desc => { + if (seenTypes.has(desc.id)) { + return false; + } + seenTypes.add(desc.id); + return true; + }); + } + + private recalculateLinkOffsets( + parent: ts.Node, + links: TypeLink[], + ): TypeLink[] { + const parentStart = parent.getStart(); + return links.map(link => { + const [start, end] = link.location; + return { + ...link, + location: [start - parentStart, end - parentStart], + }; + }); + } +} diff --git a/packages/docgen/src/docgen/ApiDocPrinter.ts b/packages/docgen/src/docgen/ApiDocPrinter.ts new file mode 100644 index 0000000000..5824047ef7 --- /dev/null +++ b/packages/docgen/src/docgen/ApiDocPrinter.ts @@ -0,0 +1,112 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { execSync } from 'child_process'; +import MarkdownPrinter from './MarkdownPrinter'; +import sortSelector from './sortSelector'; +import { ApiDoc } from './types'; + +const GH_BASE_URL = 'https://github.com/spotify/backstage'; + +const COMMIT_SHA = + process.env.COMMIT_SHA || execSync('git rev-parse HEAD').toString('utf8'); + +/** + * The ApiDocPrinter takes a ApiDoc data structure, typically generated by an ApiDocGenerator, + * and prints it out as a markdown doc with custom code highlighting and links. + */ +export default class ApiDocPrinter { + printerFactory: () => MarkdownPrinter; + + constructor(printerFactory: () => MarkdownPrinter) { + this.printerFactory = printerFactory; + } + + mkTypeLink({ file, lineInFile }: { file: string; lineInFile: number }) { + const text = `${file}:${lineInFile}`; + const href = `${GH_BASE_URL}/blob/${COMMIT_SHA}/${file}#L${lineInFile}`; + return `[${text}](${href}){:target="_blank"}`; + } + + print(apiDoc: ApiDoc): Buffer { + const printer = this.printerFactory(); + + // Remove line numbers from codeblocks + printer.style('.linenodiv{ display: none }'); + + printer.header(1, `shared/apis/${apiDoc.id}`); + + const ifInfo = apiDoc.interfaceInfo; + + if (ifInfo.docs.length) { + for (const doc of ifInfo.docs) { + printer.text(doc); + } + } else { + printer.paragraph(apiDoc.description); + } + + printer.header(2, 'API Interface'); + printer.paragraph( + `The API interface type is defined at ${this.mkTypeLink(ifInfo)} as ${ + ifInfo.name + }.`, + ); + printer.paragraph('All members of the interface are listed below.'); + + for (const member of ifInfo.members) { + printer.header( + 3, + `${member.name}${member.type === 'method' ? '()' : ''}`, + member.path, + ); + + for (const doc of member.docs) { + printer.text(doc); + } + + printer.codeWithLinks(member); + } + + if (ifInfo.dependentTypes.length) { + printer.header(2, 'Types'); + } + for (const type of ifInfo.dependentTypes + .slice() + .sort(sortSelector(x => x.name))) { + printer.header(3, `${type.name}`, type.path); + + for (const doc of type.docs) { + printer.text(doc); + } + printer.codeWithLinks(type); + + printer.paragraph(`Defined at ${this.mkTypeLink(type)}.`); + + const usageLinks = [...ifInfo.members, ...ifInfo.dependentTypes] + .filter(member => { + return member.links.some(link => link.id === type.id); + }) + .map(({ name, path }) => `[${name}](#${path})`); + + if (usageLinks.length) { + printer.paragraph(`Referenced by: ${usageLinks.join(', ')}.`); + } + } + + return printer.toBuffer(); + } +} diff --git a/packages/docgen/src/docgen/MarkdownPrinter.ts b/packages/docgen/src/docgen/MarkdownPrinter.ts new file mode 100644 index 0000000000..c33ecc76a8 --- /dev/null +++ b/packages/docgen/src/docgen/MarkdownPrinter.ts @@ -0,0 +1,136 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sortSelector from './sortSelector'; +import { Highlighter, TypeLink } from './types'; + +/** + * The MarkdownPrinter is a helper class for building markdown documents. + */ +export default class MarkdownPrinter { + private str: string = ''; + private readonly highlighter: Highlighter; + + constructor(highlighter: Highlighter) { + this.highlighter = highlighter; + } + + private line(line: string = '') { + this.str += `${line}\n`; + } + + text(text: string) { + this.line(text); + this.line(); + } + + style(style: string) { + this.line(''); + } + + header(level: number, text: string, id?: string) { + this.line(`${'#'.repeat(level)} ${text}${id ? ` {#${id}}` : ''}`); + this.line(); + } + + paragraph(text: string) { + this.line( + text + .trim() + .split('\n') + .map(line => line.trim()) + .join('\n'), + ); + this.line(); + } + + code(syntax: string = '', block: string) { + this.line(`\`\`\`${syntax}`); + this.line(block); + this.line('```'); + this.line(); + } + + codeWithLinks({ text, links = [] }: { text: string; links?: TypeLink[] }) { + this.line('
'); + this.line( + '
', + ); + this.line('
');
+    this.line(this.formatWithLinks({ text, links }));
+    this.line('
'); + this.line('
'); + this.line('
'); + } + + private escapeText: (text: string) => string = (() => { + const escapes: { [char in string]: string } = { + '&': '&', + '<': '<', + '>': '>', + }; + + return (text: string) => text.replace(/[&<>]/g, char => escapes[char]); + })(); + + private formatWithLinks({ + text, + links = [], + }: { + text: string; + links?: TypeLink[]; + }): string { + const sortedLinks = links.slice().sort(sortSelector(x => x.location[0])); + + sortedLinks.reduce((lastEnd, link) => { + if (link.location[0] <= lastEnd) { + throw new Error( + `overlapping link detected for ${link.path}, ${link.location}`, + ); + } + return link.location[1]; + }, -1); + + const parts: Array<{ text: string; path?: string }> = []; + + const end = sortedLinks.reduce((prev, link) => { + const [start, end] = link.location; + parts.push( + { text: text.slice(prev, start) }, + { text: text.slice(start, end), path: link.path }, + ); + return end; + }, 0); + + parts.push({ text: text.slice(end) }); + + return parts + .map(part => { + if (part.path) { + return `${this.escapeText(part.text)}`; + } else { + return this.highlighter.highlight(this.escapeText(part.text)); + } + }) + .join(''); + } + + toBuffer(): Buffer { + return Buffer.from(this.str, 'utf8'); + } +} diff --git a/packages/docgen/src/docgen/TypeLocator.test.ts b/packages/docgen/src/docgen/TypeLocator.test.ts new file mode 100644 index 0000000000..651fbf13b2 --- /dev/null +++ b/packages/docgen/src/docgen/TypeLocator.test.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; +import TypeLocator from './TypeLocator'; +import { createMemProgram } from './testUtils'; + +describe('TypeLocator', () => { + it('should find a default export', () => { + const program = createMemProgram('export default class MyApi {}'); + + const typeLocator = TypeLocator.fromProgram(program); + + const apiType = typeLocator.getExportedType('/mem/index.ts'); + expect(apiType.symbol.name).toBe('default'); + const [declaration] = apiType.symbol.declarations; + expect(declaration.kind).toBe(ts.SyntaxKind.ClassDeclaration); + expect((declaration as ts.ClassDeclaration).name).toBeDefined(); + expect((declaration as ts.ClassDeclaration).name!.text).toBe('MyApi'); + }, 10000); + + it('should find api instance export', () => { + const program = createMemProgram( + ` + import MyApi from './type'; + + type MyApiType = {}; + + export const myApi = new MyApi(); + `, + { + '/mem/type.ts': 'export default class MyApi {}', + }, + ); + + const typeLocator = TypeLocator.fromProgram(program); + + const { apiInstances } = typeLocator.findExportedInstances({ + apiInstances: typeLocator.getExportedType('/mem/type.ts'), + }); + + expect(apiInstances.length).toBe(1); + const [apiInstance] = apiInstances; + expect(apiInstance.name).toBe('myApi'); + expect(apiInstance.source.fileName).toBe('/mem/index.ts'); + expect(apiInstance.args).toEqual([]); + + expect(apiInstance.typeArgs.length).toBe(1); + const [typeArg] = apiInstance.typeArgs; + expect(typeArg.kind).toBe(ts.SyntaxKind.TypeReference); + expect((typeArg as ts.TypeReferenceNode).typeName.getText()).toBe( + 'MyApiType', + ); + }); +}); diff --git a/packages/docgen/src/docgen/TypeLocator.ts b/packages/docgen/src/docgen/TypeLocator.ts new file mode 100644 index 0000000000..8342548e01 --- /dev/null +++ b/packages/docgen/src/docgen/TypeLocator.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; +import { ExportedInstance } from './types'; + +/** + * The TypeLocator is used to extract typescrint Type structures from a compiled program, + * as well as finding locations where types are used in according to a matching pattern. + * + * This is used to e.g. find exported APIs that we should generate documentation for. + */ +export default class TypeLocator { + private readonly checker: ts.TypeChecker; + private readonly program: ts.Program; + + static fromProgram(program: ts.Program) { + return new TypeLocator(program.getTypeChecker(), program); + } + + constructor(checker: ts.TypeChecker, program: ts.Program) { + this.checker = checker; + this.program = program; + } + + getExportedType(path: string, exportedName: string = 'default'): ts.Type { + const source = this.program.getSourceFile(path); + if (!source) { + throw new Error(`Source not found for path '${path}'`); + } + const exported = this.checker.getExportsOfModule( + this.checker.getSymbolAtLocation(source)!, + ); + const [symbol] = exported.filter(e => e.name === exportedName); + if (!symbol) { + throw new Error(`No export '${exportedName}' found in ${path}`); + } + const type = this.checker.getTypeOfSymbolAtLocation(symbol, source); + return type; + } + + findExportedInstances( + typeLookupTable: { [key in T]: ts.Type }, + ): { [key in T]: ExportedInstance[] } { + const docMap = new Map(); + for (const type of Object.values(typeLookupTable)) { + docMap.set(type, []); + } + + this.program.getSourceFiles().forEach(source => { + ts.forEachChild(source, node => { + const decl = this.getExportedConstructorDeclaration(node); + if (!decl || !docMap.has(decl.constructorType)) { + return; + } + + docMap.get(decl.constructorType)!.push({ + node, + name: decl.name, + source, + args: Array.from(decl.initializer.arguments || []), + typeArgs: Array.from(decl.initializer.typeArguments || []), + }); + }); + }); + + const result: { [key in T]?: ExportedInstance[] } = {}; + + for (const key in typeLookupTable) { + if (typeLookupTable.hasOwnProperty(key)) { + const type = typeLookupTable[key]; + result[key] = docMap.get(type)!; + } + } + + return result as { [key in T]: ExportedInstance[] }; + } + + private getExportedConstructorDeclaration( + node: ts.Node, + ): + | { constructorType: ts.Type; initializer: ts.NewExpression; name: string } + | undefined { + if (!ts.isVariableStatement(node)) { + return; + } + if ( + !node.modifiers || + !node.modifiers.some(mod => mod.kind === ts.SyntaxKind.ExportKeyword) + ) { + return; + } + const { declarations } = node.declarationList; + if (declarations.length !== 1) { + return; + } + + const [declaration] = declarations; + const { initializer, name } = declaration; + + if (!initializer || !name) { + return; + } + if (!ts.isNewExpression(initializer)) { + return; + } + if (!ts.isIdentifier(name)) { + return; + } + + const constructorType = this.checker.getTypeAtLocation( + initializer.expression, + ); + return { constructorType, initializer, name: name.text }; + } +} diff --git a/packages/docgen/src/docgen/TypescriptHighlighter.ts b/packages/docgen/src/docgen/TypescriptHighlighter.ts new file mode 100644 index 0000000000..a16c9a2c28 --- /dev/null +++ b/packages/docgen/src/docgen/TypescriptHighlighter.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Highlighter } from './types'; +import { flatMap } from './flatMap'; + +// Simple syntax highlighter that mimics hilite +export default class TypescriptHighlighter implements Highlighter { + private static readonly basicTypes = [ + 'boolean', + 'number', + 'string', + 'Array', + 'object', + 'Record', + 'Set', + 'Map', + 'true', + 'false', + 'null', + 'undefined', + 'void', + 'Promise', + 'any', + '[0-9\\.]+', + ]; + + // List of highlightings to apply, each with a match regex and the style that should be applied + private static readonly highlighters = [ + [/(\/\*\*?(?:.|\n)+?\*\/)/g, 'color: #60a0b0; font-style: italic'], // block comment + [/(\/\/.*)/gm, 'color: #60a0b0; font-style: italic'], // line comment + [/('[^']*?'|"[^"]*?")/g, 'color: #4070a0'], // string literals + [ + new RegExp( + `\\b(${TypescriptHighlighter.basicTypes.join('|')})\\b(?!:|,)`, + 'g', + ), + 'color: #902000', + ], // basic types + [ + /^((?:export\s)?(?:type\s)?(?:interface\s)?)/g, + 'color: #007020; font-weight: bold', + ], // keywords + ] as readonly [RegExp, string][]; + + highlight(text: string): string { + // Each part is either plain text that can be highlighted or text that is already highlighted + type HighlightPart = { text: string; highlighted?: boolean }; + + const painter = (regex: RegExp, style: string) => (part: HighlightPart) => { + if (part.highlighted) { + return [part]; + } + // Apply highlighting to all matches of the regex by splitting into parts + return part.text.split(regex).map((text, index) => { + // Odd parts are the ones that matched the regex and should be highlighted. + if (index % 2 === 1) { + return { + text: `${text}`, + highlighted: true, + }; + } + return { text }; + }); + }; + + // Order here is important, e.g. comments must be first to avoid string literals inside comments being highlighted + return TypescriptHighlighter.highlighters + .reduce((parts, highlighter) => flatMap(parts, painter(...highlighter)), [ + { text }, + ]) + .map(({ text }) => text) + .join(''); + } +} diff --git a/packages/docgen/src/docgen/flatMap.test.ts b/packages/docgen/src/docgen/flatMap.test.ts new file mode 100644 index 0000000000..6186863039 --- /dev/null +++ b/packages/docgen/src/docgen/flatMap.test.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { flatMap } from './flatMap'; + +describe('flatMap', () => { + it('should flat map', () => { + expect(flatMap([], x => x)).toEqual([]); + expect(flatMap([[]], x => x)).toEqual([]); + // This simple implementation only flattens once + expect(flatMap([[[]]], x => x)).toEqual([[]]); + + expect(flatMap([[1, 2], 3, [4, 5]], x => x)).toEqual([1, 2, 3, 4, 5]); + }); +}); diff --git a/packages/docgen/src/docgen/flatMap.ts b/packages/docgen/src/docgen/flatMap.ts new file mode 100644 index 0000000000..bdea672a41 --- /dev/null +++ b/packages/docgen/src/docgen/flatMap.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function flatMap(array: T[], func: (t: T) => R[] | R): R[] { + const out = array.map(func); + return ([] as R[]).concat.apply([], out as R[]); +} diff --git a/packages/docgen/src/docgen/sortSelector.test.ts b/packages/docgen/src/docgen/sortSelector.test.ts new file mode 100644 index 0000000000..a3da3e2950 --- /dev/null +++ b/packages/docgen/src/docgen/sortSelector.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import sortSelector from './sortSelector'; + +describe('sortSelector', () => { + it('should stable sort', () => { + const arr = [ + [3, 1], + [1, 2], + [1, 1], + [1, 3], + [2, 1], + ]; + + const sortedByFirst = arr.slice().sort(sortSelector(([first]) => first)); + expect(sortedByFirst).toEqual([ + [1, 2], + [1, 1], + [1, 3], + [2, 1], + [3, 1], + ]); + + const sortedBySecond = arr + .slice() + .sort(sortSelector(([, second]) => second)); + expect(sortedBySecond).toEqual([ + [3, 1], + [1, 1], + [2, 1], + [1, 2], + [1, 3], + ]); + }); +}); diff --git a/packages/docgen/src/docgen/sortSelector.ts b/packages/docgen/src/docgen/sortSelector.ts new file mode 100644 index 0000000000..1ba7a7918d --- /dev/null +++ b/packages/docgen/src/docgen/sortSelector.ts @@ -0,0 +1,34 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The sortSelector is a utility that makes a sort function by selecting the value to sort on with a lambda. + */ +export default function sortSelector( + selector: (x: T) => any, +): (a: T, b: T) => -1 | 1 | 0 { + return (a: T, b: T) => { + let aV = selector(a); + let bV = selector(b); + if (aV < bV) { + return -1; + } else if (aV > bV) { + return 1; + } else { + return 0; + } + }; +} diff --git a/packages/docgen/src/docgen/testUtils.ts b/packages/docgen/src/docgen/testUtils.ts new file mode 100644 index 0000000000..552776016a --- /dev/null +++ b/packages/docgen/src/docgen/testUtils.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; + +export function createMemProgram( + indexSource: string, + otherSourceFiles: { [fileName in string]: string } = {}, +): ts.Program { + const rootDir = '/mem'; + + const options = { noEmit: true }; + const baseHost = ts.createCompilerHost(options); + const files: { [fileName in string]: string } = { + [`${rootDir}/index.ts`]: indexSource, + ...otherSourceFiles, + }; + + // Custom compiler hosts that reads from a map of in-memory files, but + // falls back to reading from disc for ts libs etc. + const compilerHost: ts.CompilerHost = { + ...baseHost, + readFile(fileName): string | undefined { + if (fileName in files) { + return files[fileName]; + } + return baseHost.readFile(fileName); + }, + getCurrentDirectory(): string { + return rootDir; + }, + directoryExists(dir) { + if (dir === rootDir) { + return true; + } + if (baseHost.directoryExists) { + return baseHost.directoryExists(dir); + } + return false; + }, + fileExists(fileName: string): boolean { + return !!files[fileName] || baseHost.fileExists(fileName); + }, + getSourceFile(fileName, ...rest): ts.SourceFile | undefined { + const file = files[fileName]; + if (file) { + return ts.createSourceFile(fileName, file, ts.ScriptTarget.ES2017); + } + return baseHost.getSourceFile(fileName, ...rest); + }, + }; + + return ts.createProgram(['/mem/index.ts'], options, compilerHost); +} diff --git a/packages/docgen/src/docgen/types.ts b/packages/docgen/src/docgen/types.ts new file mode 100644 index 0000000000..df9a5b93a0 --- /dev/null +++ b/packages/docgen/src/docgen/types.ts @@ -0,0 +1,96 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ts from 'typescript'; + +/** + * A TypeLink is a link to a different type. + */ +export type TypeLink = { + // The ID of the linked type + id: number; + // The path to the type from the project root + path: string; + // The name of the type, to display + name: string; + // The location of the type name as it appears in it's parent text. + location: readonly [number, number]; +}; + +/** + * TypeInfo describes a Typescript Type. + */ +export type TypeInfo = { + id: number; + name: string; + path: string; + file: string; + lineInFile: number; + text: string; + docs: string[]; + links: TypeLink[]; + children: TypeInfo[]; +}; + +/** + * FieldInfo describes a property or method in a documented API. + */ +export type FieldInfo = { + type: 'prop' | 'method'; + name: string; + path: string; + text: string; + docs: string[]; + links: TypeLink[]; +}; + +/** + * InterfaceInfo describes the type of a documented API. + */ +export type InterfaceInfo = { + name: string; + docs: string[]; + file: string; + lineInFile: number; + members: Array; + dependentTypes: TypeInfo[]; +}; + +/** + * ApiDoc describes a documented API. + */ +export type ApiDoc = { + id: string; + name: string; + source: ts.SourceFile; + description: string; + interfaceInfo: InterfaceInfo; +}; + +/** + * ExportedInstance describes an expression matching `export {name} = new {Contructor}<{typeArgs}...>({args}...)` + */ +export type ExportedInstance = { + node: ts.Node; + name: string; + source: ts.SourceFile; + args: Array; + typeArgs: Array; +}; + +export type Highlighter = { + highlight(test: string): string; +}; diff --git a/packages/docgen/src/index.ts b/packages/docgen/src/index.ts new file mode 100644 index 0000000000..d57c49f9f8 --- /dev/null +++ b/packages/docgen/src/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import './compiler';