Google Cloud Storage working with TechDocs (publish and fetch)
Plus 1. Introduce ncessary configs to connect with storage 2. Introduce config to prefer CI build of docs, or techdocs-backend to build and publish them. 3. Write guide how to use Google Cloud Storage with TechDocs 4. Add a TechDocs Configuration reference page in docucmentation
This commit is contained in:
Himanshu Mishra
@@ -39,14 +39,16 @@
|
||||
"@backstage/backend-common": "^0.3.2",
|
||||
"@backstage/catalog-model": "^0.3.1",
|
||||
"@backstage/config": "^0.1.1",
|
||||
"@google-cloud/storage": "^5.6.0",
|
||||
"@types/dockerode": "^2.5.34",
|
||||
"@types/express-serve-static-core": "^4.17.14",
|
||||
"@types/klaw": "^3.0.1",
|
||||
"cross-fetch": "^3.0.6",
|
||||
"dockerode": "^3.2.1",
|
||||
"express": "^4.17.1",
|
||||
"fs-extra": "^9.0.1",
|
||||
"git-url-parse": "^11.4.0",
|
||||
"js-yaml": "^3.14.0",
|
||||
"klaw": "^3.0.0",
|
||||
"mock-fs": "^4.13.0",
|
||||
"nodegit": "^0.27.0",
|
||||
"winston": "^3.2.1"
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import path from 'path';
|
||||
import express from 'express';
|
||||
import walk from 'klaw';
|
||||
import { Storage, UploadResponse } from '@google-cloud/storage';
|
||||
import { Logger } from 'winston';
|
||||
import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
import { Config } from '@backstage/config';
|
||||
import { getHeadersForFileExtension, supportedFileType } from './helpers';
|
||||
import { PublisherBase, PublisherBaseParams } from './types';
|
||||
|
||||
export class GoogleGCSPublish implements PublisherBase {
|
||||
static fromConfig(config: Config, logger: Logger): PublisherBase {
|
||||
let pathToKey = '';
|
||||
let projectId = '';
|
||||
let bucketName = '';
|
||||
try {
|
||||
pathToKey = config.getString('techdocs.publisher.google.pathToKey');
|
||||
projectId = config.getString('techdocs.publisher.google.projectId');
|
||||
bucketName = config.getString('techdocs.publisher.google.bucketName');
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
"Since techdocs.publisher.type is set to 'google_gcs' in your app config, " +
|
||||
'pathToKey, projectId and bucketName are required in techdocs.publisher.google ' +
|
||||
'required to authenticate with Google Cloud Storage.',
|
||||
);
|
||||
}
|
||||
|
||||
const storageClient = new Storage({
|
||||
projectId: projectId,
|
||||
keyFilename: pathToKey,
|
||||
});
|
||||
|
||||
// Check if the defined bucket exists. Being able to connect means the configuration is good
|
||||
// and the storage client will work.
|
||||
storageClient
|
||||
.bucket(bucketName)
|
||||
.getMetadata()
|
||||
.then(() => {
|
||||
logger.info(
|
||||
`Successfully connected to the GCS bucket ${bucketName} in the GCP project ${projectId}.`,
|
||||
);
|
||||
})
|
||||
.catch(reason => {
|
||||
logger.error(
|
||||
`Could not retrieve metadata about the GCS bucket ${bucketName} in the GCP project ${projectId}. ` +
|
||||
'Make sure the GCP project and the bucket exists and the access key located at the path ' +
|
||||
"techdocs.publisher.google.pathToKey defined in app config has the role 'Storage Object Creator'. " +
|
||||
'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage',
|
||||
);
|
||||
throw new Error(`from GCS client library: ${reason.message}`);
|
||||
});
|
||||
|
||||
return new GoogleGCSPublish(storageClient, bucketName, logger);
|
||||
}
|
||||
|
||||
constructor(
|
||||
private readonly storageClient: Storage,
|
||||
private readonly bucketName: string,
|
||||
private readonly logger: Logger,
|
||||
) {
|
||||
this.storageClient = storageClient;
|
||||
this.bucketName = bucketName;
|
||||
this.logger = logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload all the files from the generated `directory` to the GCS bucket.
|
||||
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
|
||||
*/
|
||||
publish({ entity, directory }: PublisherBaseParams): Promise<{}> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Path of all files to upload, relative to the root of the source directory
|
||||
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
|
||||
const allFilesToUpload: Array<string> = [];
|
||||
|
||||
// Iterate on all the files in the directory and its sub-directories
|
||||
walk(directory)
|
||||
.on('data', (item: walk.Item) => {
|
||||
// GCS manages creation of parent directories if they do not exist.
|
||||
// So collecting path of only the files is good enough.
|
||||
if (item.stats.isFile()) {
|
||||
// Remove the absolute path prefix of the source directory
|
||||
const relativeFilePath = item.path.replace(`${directory}/`, '');
|
||||
allFilesToUpload.push(relativeFilePath);
|
||||
}
|
||||
})
|
||||
.on('error', (err: Error, item: walk.Item) => {
|
||||
const errorMessage = `Unable to read file at ${item.path}. Error ${err.message}`;
|
||||
this.logger.error(errorMessage);
|
||||
reject(errorMessage);
|
||||
})
|
||||
.on('end', () => {
|
||||
// 'end' event happens when all the files have been read.
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
allFilesToUpload.forEach(filePath => {
|
||||
const source = path.join(directory, filePath); // Local file absolutely path
|
||||
const destination = `${entityRootDir}/${filePath}`; // GCS Bucket file relative path
|
||||
this.storageClient
|
||||
.bucket(this.bucketName)
|
||||
.upload(source, { destination })
|
||||
.then(
|
||||
(uploadResp: UploadResponse) => ({
|
||||
fileName: destination,
|
||||
status: uploadResp[0],
|
||||
}),
|
||||
(err: Error) => {
|
||||
const errorMessage = `Unable to upload file ${destination} to GCS. Error ${err.message}`;
|
||||
this.logger.error(errorMessage);
|
||||
reject(errorMessage);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
this.logger.info(
|
||||
`Successfully uploaded all the generated files for Entity ${entityRootDir}. Total number of files: ${allFilesToUpload.length}`,
|
||||
);
|
||||
resolve({});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
|
||||
|
||||
const fileStreamChunks: Array<any> = [];
|
||||
this.storageClient
|
||||
.bucket(this.bucketName)
|
||||
.file(`${entityRootDir}/techdocs_metadata.json`)
|
||||
.createReadStream()
|
||||
.on('error', err => {
|
||||
this.logger.error(err.message);
|
||||
reject(err.message);
|
||||
})
|
||||
.on('data', chunk => {
|
||||
fileStreamChunks.push(chunk);
|
||||
})
|
||||
.on('end', () => {
|
||||
const techdocsMetadataJson = Buffer.concat(
|
||||
fileStreamChunks,
|
||||
).toString();
|
||||
resolve(techdocsMetadataJson);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Express route middleware to serve static files on a route in techdocs-backend.
|
||||
*/
|
||||
docsRouter(): express.Handler {
|
||||
return (req, res) => {
|
||||
// Trim the leading forward slash
|
||||
// filePath example - /default/Component/documented-component/index.html
|
||||
const filePath = req.path.replace(/^\//, '');
|
||||
|
||||
// Files with different extensions (CSS, HTML) need to be served with different headers
|
||||
const fileExtension = filePath.split('.')[filePath.split('.').length - 1];
|
||||
const responseHeaders = getHeadersForFileExtension(
|
||||
fileExtension as supportedFileType,
|
||||
);
|
||||
|
||||
const fileStreamChunks: Array<any> = [];
|
||||
this.storageClient
|
||||
.bucket(this.bucketName)
|
||||
.file(filePath)
|
||||
.createReadStream()
|
||||
.on('error', err => {
|
||||
this.logger.error(err.message);
|
||||
res.send(err.message);
|
||||
})
|
||||
.on('data', chunk => {
|
||||
fileStreamChunks.push(chunk);
|
||||
})
|
||||
.on('end', () => {
|
||||
const fileContent = Buffer.concat(fileStreamChunks).toString();
|
||||
// Inject response headers
|
||||
for (const [headerKey, headerValue] of Object.entries(
|
||||
responseHeaders,
|
||||
)) {
|
||||
res.setHeader(headerKey, headerValue);
|
||||
}
|
||||
|
||||
res.send(fileContent);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A helper function which checks if index.html of an Entity's docs site is available. This
|
||||
* can be used to verify if there are any pre-generated docs available to serve.
|
||||
*/
|
||||
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
|
||||
return new Promise(resolve => {
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
this.storageClient
|
||||
.bucket(this.bucketName)
|
||||
.file(`${entityRootDir}/index.html`)
|
||||
.createReadStream()
|
||||
.on('error', () => {
|
||||
resolve(false);
|
||||
})
|
||||
.on('data', () => {
|
||||
resolve(true);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright 2020 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export type supportedFileType = 'html' | 'css';
|
||||
|
||||
export type responseHeadersType = {
|
||||
'Content-Type': string;
|
||||
};
|
||||
|
||||
export const getHeadersForFileExtension = (
|
||||
fileType: supportedFileType,
|
||||
): responseHeadersType => {
|
||||
const headersCommon = {
|
||||
'Content-Type': 'text/plain',
|
||||
};
|
||||
const headersHTML = {
|
||||
...headersCommon,
|
||||
'Content-Type': 'text/html; charset=UTF-8',
|
||||
};
|
||||
|
||||
const headersCSS = {
|
||||
...headersCommon,
|
||||
'Content-Type': 'text/css; charset=UTF-8',
|
||||
};
|
||||
|
||||
switch (fileType) {
|
||||
case 'html':
|
||||
return headersHTML;
|
||||
case 'css':
|
||||
return headersCSS;
|
||||
default:
|
||||
return headersCommon;
|
||||
}
|
||||
};
|
||||
@@ -13,12 +13,11 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import express from 'express';
|
||||
import * as expressServeStaticCore from 'express-serve-static-core'; // Type from express library
|
||||
import fetch from 'cross-fetch';
|
||||
import express from 'express';
|
||||
import fs from 'fs-extra';
|
||||
import { Logger } from 'winston';
|
||||
import { EntityName } from '@backstage/catalog-model';
|
||||
import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
import {
|
||||
resolvePackagePath,
|
||||
PluginEndpointDiscovery,
|
||||
@@ -92,9 +91,7 @@ export class LocalPublish implements PublisherBase {
|
||||
});
|
||||
}
|
||||
|
||||
fetchTechDocsMetadata(
|
||||
entityName: EntityName,
|
||||
): Promise<{ techdocsMetadataJson: string }> {
|
||||
fetchTechDocsMetadata(entityName: EntityName): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => {
|
||||
const storageUrl = new URL(
|
||||
@@ -102,8 +99,8 @@ export class LocalPublish implements PublisherBase {
|
||||
techdocsApiUrl,
|
||||
).toString();
|
||||
|
||||
const path = `${entityName.kind}/${entityName.namespace}/${entityName.name}`;
|
||||
const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`;
|
||||
const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
|
||||
const metadataURL = `${storageUrl}/${entityRootDir}/techdocs_metadata.json`;
|
||||
fetch(metadataURL)
|
||||
.then(response =>
|
||||
response
|
||||
@@ -111,18 +108,37 @@ export class LocalPublish implements PublisherBase {
|
||||
.then(techdocsMetadataJson => resolve(techdocsMetadataJson))
|
||||
.catch(err => {
|
||||
reject(
|
||||
`Unable to parse metadata JSON for ${path}. Error: ${err}`,
|
||||
`Unable to parse metadata JSON for ${entityRootDir}. Error: ${err}`,
|
||||
);
|
||||
}),
|
||||
)
|
||||
.catch(err => {
|
||||
reject(`Unable to fetch metadata for ${path}. Error ${err}`);
|
||||
reject(
|
||||
`Unable to fetch metadata for ${entityRootDir}. Error ${err}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
docsRouter(): expressServeStaticCore.Handler {
|
||||
docsRouter(): express.Handler {
|
||||
return express.static(staticDocsDir);
|
||||
}
|
||||
|
||||
async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
|
||||
return new Promise(resolve => {
|
||||
this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => {
|
||||
const storageUrl = new URL(
|
||||
new URL(this.config.getString('techdocs.storageUrl')).pathname,
|
||||
techdocsApiUrl,
|
||||
).toString();
|
||||
|
||||
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
|
||||
const indexHtmlUrl = `${storageUrl}/${entityRootDir}/index.html`;
|
||||
fetch(indexHtmlUrl)
|
||||
.then(() => resolve(true))
|
||||
.catch(() => resolve(false));
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common';
|
||||
|
||||
import { PublisherType, PublisherBase } from './types';
|
||||
import { LocalPublish } from './local';
|
||||
import { GoogleGCSPublish } from './googleStorage';
|
||||
|
||||
/**
|
||||
* Factory class to create a TechDocs publisher based on defined publisher type in app config.
|
||||
@@ -37,7 +38,7 @@ export class Publisher {
|
||||
switch (publisherType) {
|
||||
case 'google_gcs':
|
||||
logger.info('Creating Google Storage Bucket publisher for TechDocs');
|
||||
return new LocalPublish(config, logger, discovery);
|
||||
return GoogleGCSPublish.fromConfig(config, logger);
|
||||
case 'local':
|
||||
logger.info('Creating Local publisher for TechDocs');
|
||||
return new LocalPublish(config, logger, discovery);
|
||||
|
||||
@@ -14,9 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Entity, EntityName } from '@backstage/catalog-model';
|
||||
// import serveStatic from 'serve-static';
|
||||
// import express from 'express';
|
||||
import * as express from 'express-serve-static-core';
|
||||
import express from 'express';
|
||||
|
||||
/**
|
||||
* Key for all the different types of TechDocs publishers that are supported.
|
||||
@@ -49,12 +47,15 @@ export interface PublisherBase {
|
||||
* Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc.
|
||||
* This API uses the techdocs_metadata.json file that co-exists along with the generated docs.
|
||||
*/
|
||||
fetchTechDocsMetadata(
|
||||
entityName: EntityName,
|
||||
): Promise<{ techdocsMetadataJson: string }>;
|
||||
fetchTechDocsMetadata(entityName: EntityName): Promise<string>;
|
||||
|
||||
/**
|
||||
*
|
||||
* Route middleware to serve static documentation files for an entity.
|
||||
*/
|
||||
docsRouter(): express.Handler;
|
||||
|
||||
/**
|
||||
* Check if the index.html is present for the Entity at the Storage location.
|
||||
*/
|
||||
hasDocsBeenGenerated(entityName: Entity): Promise<boolean>;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user