paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 39 Packages Projects Releases Wiki Activity

adjusting tests to use msw and other changes requested in pr

Browse Source 
Signed-off-by: Tracey <tradcliffe@expediagroup.com>
This commit is contained in:
Tracey
2023-03-16 15:41:52 -07:00
parent b20e2e16fd
commit 18e6ef9e88
8 changed files with 466 additions and 448 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/new-pigs-jam.md
+2 -2
View File
@@ -1,5 +1,5 @@
---
'@backstage/plugin-scaffolder-backend-module-confluence-to-markdowwn': minor
'@backstage/plugin-scaffolder-backend-module-confluence-to-markdown': minor
---
Created new action for converting confluence docs to Markdown, to be pushed to GitHub repos onboarded onto Techdocs
Created new action for converting confluence docs to Markdown, to be pushed to GitHub repos configured for Techdocs
plugins/scaffolder-backend-module-confluence-to-markdowwn/README.md
+9 -11
View File
@@ -1,4 +1,4 @@
# @backstage/plugin-scaffolder-backend-module-confluence-to-markdowwn
# @backstage/plugin-scaffolder-backend-module-confluence-to-markdown
Welcome to the `transform:confluence-to-markdown` action for the `scaffolder-backend`.
@@ -10,7 +10,7 @@ You need to configure the action in your backend:
```bash
# From your Backstage root directory
yarn add --cwd packages/backend @backstage/plugin-scaffolder-backend-module-confluence-to-markdowwn
yarn add --cwd packages/backend @backstage/plugin-scaffolder-backend-module-confluence-to-markdown
```
Configure the action:
@@ -21,7 +21,7 @@ Configure the action:
import { createBuiltinActions } from '@backstage/plugin-scaffolder-backend';
import { ScmIntegrations } from '@backstage/integration';
import { createConfluenceToMarkdownAction } from '@backstage/plugin-scaffolder-backend-module-confluence-to-markdowwn';
import { createConfluenceToMarkdownAction } from '@backstage/plugin-scaffolder-backend-module-confluence-to-markdown';
export default async function createPlugin(
env: PluginEnvironment,
@@ -58,16 +58,15 @@ export default async function createPlugin(
You will also need an access token for authorization with `Read` permissions. You can create a Personal Access Token (PAT) in confluence and add the PAT to your `app-config.yaml`
```
```yaml
confluence:
baseUrl: ${CONFLUENCE_BASE_URL}
token: ${CONFLUENCE_TOKEN}
```
After that you can use the action in your template:
```
```yaml
apiVersion: scaffolder.backstage.io/v1beta3
kind: Template
metadata:
@@ -85,7 +84,7 @@ spec:
properties:
confluenceUrls:
type: array
description: Urls for confluence doc to be converted to markdown. In format <CONFLUENCE_BASE_URL>/display/<SPACEKEY>/<PAGE+TITLE> or <CONFLUENCE_BASE_URL>/pages/viewpage.action?spacekey=<SPACEKEY>&title=<PAGE_TITLE>
description: Urls for confluence doc to be converted to markdown. In format <CONFLUENCE_BASE_URL>/display/<SPACEKEY>/<PAGE+TITLE>
items:
type: string
default: confluence url
@@ -95,8 +94,8 @@ spec:
maxItems: 5
repoUrl:
type: string
title: GitHub url mkdocs.yaml link
description: The GitHub repo url to your mkdocs.yaml file. Example <https://github.com/blob/master/mkdocs.yml>
title: GitHub URL mkdocs.yaml link
description: The GitHub repo URL to your mkdocs.yaml file. Example <https://github.com/blob/master/mkdocs.yml>
steps:
- id: create-docs
name: Get markdown file created and update markdown.yaml file
@@ -118,9 +117,8 @@ spec:
input:
prUrl: ${{ steps['publish'].output.remoteUrl }}
forceAdmin: true
```
Replace `<GITHUB_BASE_URL>` with your GitHub url without `https://`.
You can find a list of all registred actions including their parameters at the /create/actions route in your Backstage application.
You can find a list of all registered actions including their parameters at the /create/actions route in your Backstage application.
plugins/scaffolder-backend-module-confluence-to-markdowwn/package.json
+5 -2
View File
@@ -1,6 +1,6 @@
{
"name": "@backstage/plugin-scaffolder-backend-module-confluence-to-markdown",
"description": "The confluence-to-markdowwn module for @backstage/plugin-scaffolder-backend",
"description": "The confluence-to-markdown module for @backstage/plugin-scaffolder-backend",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
@@ -31,16 +31,19 @@
"@backstage/plugin-scaffolder-backend": "workspace:^",
"@backstage/plugin-scaffolder-node": "workspace:^",
"@backstage/types": "workspace:^",
"fs": "^0.0.1-security",
"fs-extra": "^11.1.0",
"git-url-parse": "^13.1.0",
"jest": "^29.5.0",
"node-fetch": "2.6.7",
"node-html-markdown": "^1.3.0",
"ts-jest": "^29.0.5",
"yaml": "^2.2.1"
},
"devDependencies": {
"@backstage/cli": "workspace:^",
"@backstage/test-utils": "workspace:^",
"mock-fs": "^5.2.0",
"msw": "^1.1.0",
"os": "^0.1.2"
},
"files": [
plugins/scaffolder-backend-module-confluence-to-markdowwn/src/actions/confluence/confluenceToMarkdown.test.ts
+56 -65
View File
@@ -19,14 +19,14 @@ import { getVoidLogger } from '@backstage/backend-common';
import { UrlReader } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import { setupRequestMockHandlers } from '@backstage/test-utils';
import mockFs from 'mock-fs';
import os from 'os';
import fetch from 'node-fetch';
import type { ActionContext } from '@backstage/plugin-scaffolder-node';
import { readFile, writeFile } from 'fs-extra';
import { Readable } from 'stream';
import { readFile, writeFile, createWriteStream } from 'fs-extra';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
jest.mock('node-fetch');
jest.mock('fs-extra', () => ({
mkdirSync: jest.fn(),
readFile: jest.fn().mockResolvedValue('File contents'),
@@ -38,12 +38,15 @@ jest.mock('fs-extra', () => ({
createWriteStream: jest.fn().mockReturnValue(new PassThrough()),
writeStream: jest.fn(),
}));
const { Response } = jest.requireActual('node-fetch');
describe('transform:confluence-to-markdown', () => {
const baseUrl = `https://nodomain.confluence.com`;
const worker = setupServer();
setupRequestMockHandlers(worker);
const config = new ConfigReader({
confluence: {
baseUrl: 'https://nodomain.confluence.com/',
baseUrl: baseUrl,
token: 'fake_token',
},
});
@@ -131,25 +134,19 @@ describe('transform:confluence-to-markdown', () => {
],
};
(fetch as jest.MockedFunction<typeof fetch>)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBody), {
status: 200,
statusText: 'OK',
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBodyTwo), {
status: 200,
statusText: 'OK',
}),
)
.mockResolvedValueOnce(
new Response(Readable.from('Hello world'), {
status: 200,
statusText: 'OK',
}),
);
worker.use(
rest.get(`${baseUrl}/rest/api/content`, (_, res, ctx) =>
res(ctx.status(200, 'OK'), ctx.json(responseBody)),
),
rest.get(
`${baseUrl}/rest/api/content/4444444/child/attachment`,
(_, res, ctx) => res(ctx.status(200, 'OK'), ctx.json(responseBodyTwo)),
),
rest.get(
`${baseUrl}/download/attachments/4444444/testing.pdf`,
(_, res, ctx) => res(ctx.status(200, 'OK'), ctx.body('hello')),
),
);
const mockCreateWriteStream = jest.fn().mockReturnValue({
on: jest.fn(),
@@ -169,8 +166,7 @@ describe('transform:confluence-to-markdown', () => {
`Fetching the mkdocs.yml catalog from https://notreal.github.com/space/backstage/mkdocs.yml`,
);
expect(logger.info).toHaveBeenCalledTimes(5);
expect(fetch).toHaveBeenCalledTimes(3);
expect(createWriteStream).toHaveBeenCalledTimes(1);
expect(readFile).toHaveBeenCalledTimes(1);
expect(writeFile).toHaveBeenCalledTimes(1);
});
@@ -199,19 +195,15 @@ describe('transform:confluence-to-markdown', () => {
results: [],
};
(fetch as jest.MockedFunction<typeof fetch>)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBody), {
status: 200,
statusText: 'OK',
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBodyTwo), {
status: 200,
statusText: 'OK',
}),
);
worker.use(
rest.get(`${baseUrl}/rest/api/content`, (_, res, ctx) =>
res(ctx.status(200, 'OK'), ctx.json(responseBody)),
),
rest.get(
`${baseUrl}/rest/api/content/4444444/child/attachment`,
(_, res, ctx) => res(ctx.status(200, 'OK'), ctx.json(responseBodyTwo)),
),
);
jest.mock('fs-extra', () => ({
readFile: jest.fn().mockResolvedValue('File contents'),
@@ -225,7 +217,7 @@ describe('transform:confluence-to-markdown', () => {
);
expect(logger.info).toHaveBeenCalledTimes(5);
expect(fetch).toHaveBeenCalledTimes(2);
expect(createWriteStream).not.toHaveBeenCalled();
expect(readFile).toHaveBeenCalledTimes(1);
expect(writeFile).toHaveBeenCalledTimes(1);
});
@@ -237,13 +229,13 @@ describe('transform:confluence-to-markdown', () => {
config,
};
const responseBody = {};
(fetch as jest.MockedFunction<typeof fetch>).mockResolvedValueOnce(
new Response(JSON.stringify(responseBody), {
status: 401,
statusText: 'npot',
ok: false,
}),
worker.use(
rest.get(`${baseUrl}/rest/api/content`, (_, res, ctx) =>
res(ctx.status(401, 'nope'), ctx.json(responseBody)),
),
);
const action = createConfluenceToMarkdownAction(options);
await expect(async () => {
await action.handler(mockContext);
@@ -259,12 +251,13 @@ describe('transform:confluence-to-markdown', () => {
const responseBody = {
results: [],
};
(fetch as jest.MockedFunction<typeof fetch>).mockResolvedValueOnce(
new Response(JSON.stringify(responseBody), {
status: 200,
statusText: 'ok',
}),
worker.use(
rest.get(`${baseUrl}/rest/api/content`, (_, res, ctx) =>
res(ctx.status(200, 'OK'), ctx.json(responseBody)),
),
);
const action = createConfluenceToMarkdownAction(options);
await expect(async () => {
await action.handler(mockContext);
@@ -295,19 +288,17 @@ describe('transform:confluence-to-markdown', () => {
};
const responseBodyTwo = {};
(fetch as jest.MockedFunction<typeof fetch>)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBody), {
status: 200,
statusText: 'OK',
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify(responseBodyTwo), {
status: 404,
statusText: 'notok',
}),
);
worker.use(
rest.get(`${baseUrl}/rest/api/content`, (_, res, ctx) =>
res(ctx.status(200, 'OK'), ctx.json(responseBody)),
),
rest.get(
`${baseUrl}/rest/api/content/4444444/child/attachment`,
(_, res, ctx) =>
res(ctx.status(404, 'nope'), ctx.json(responseBodyTwo)),
),
);
const action = createConfluenceToMarkdownAction(options);
await expect(async () => {
await action.handler(mockContext);
plugins/scaffolder-backend-module-confluence-to-markdowwn/src/actions/confluence/confluenceToMarkdown.ts
+8 -13
View File
@@ -22,11 +22,9 @@ import { InputError, ConflictError } from '@backstage/errors';
import { NodeHtmlMarkdown } from 'node-html-markdown';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import YAML from 'yaml';
import {
parseFromYaml,
readFileAsString,
transformToYaml,
updateFile,
fetchConfluence,
getAndWriteAttachments,
createConfluenceVariables,
@@ -92,12 +90,9 @@ export const createConfluenceToMarkdownAction = (options: {
},
});
for (let p = 0; p < confluenceUrls.length; p++) {
const confluenceUrl = confluenceUrls[p];
for (const url of confluenceUrls) {
const { spacekey, title, titleWithSpaces } =
await createConfluenceVariables(confluenceUrl);
await createConfluenceVariables(url);
// This calls confluence to get the page html and page id
const getConfluenceDoc = await fetchConfluence(
`/rest/api/content?title=${title}&spaceKey=${spacekey}&expand=body.export_view`,
@@ -105,7 +100,7 @@ export const createConfluenceToMarkdownAction = (options: {
);
if (getConfluenceDoc.results.length === 0) {
throw new InputError(
`Could not find document ${confluenceUrl}. Please check your input.`,
`Could not find document ${url}. Please check your input.`,
);
}
// This gets attachements for the confluence page if they exist
@@ -132,7 +127,7 @@ export const createConfluenceToMarkdownAction = (options: {
// This reads mkdocs.yml file
const mkdocsFileContent = await readFileAsString(repoFileDir);
const mkdocsFile = parseFromYaml(mkdocsFileContent);
const mkdocsFile = await YAML.parse(mkdocsFileContent);
ctx.logger.info(
`Adding new file - ${titleWithSpaces} to the current mkdocs.yml file`,
);
@@ -151,14 +146,14 @@ export const createConfluenceToMarkdownAction = (options: {
);
}
}
const newTemplateFileContent = transformToYaml(mkdocsFile);
await updateFile(newTemplateFileContent, repoFileDir);
await fs.writeFile(repoFileDir, YAML.stringify(mkdocsFile));
// This grabs the confluence html and converts it to markdown and adds attachments
const html = getConfluenceDoc.results[0].body.export_view.value;
const markdownToPublish = NodeHtmlMarkdown.translate(html);
let newString: string = markdownToPublish;
productArray.map((product: string[]) => {
productArray.forEach((product: string[]) => {
const regex = product[1].includes('pdf')
? new RegExp(`(\\[.*?\\]\\()(.*?${product[0]}.*?)(\\))`, 'gi')
: new RegExp(`(\\!\\[.*?\\]\\()(.*?${product[0]}.*?)(\\))`, 'gi');
plugins/scaffolder-backend-module-confluence-to-markdowwn/src/actions/confluence/helpers.ts
+4 -18
View File
@@ -17,7 +17,6 @@
import { Config } from '@backstage/config';
import { ResponseError, ConflictError, InputError } from '@backstage/errors';
import fs from 'fs-extra';
import YAML from 'yaml';
import fetch, { Response } from 'node-fetch';
interface Links {
@@ -49,18 +48,6 @@ export const readFileAsString = async (fileDir: string) => {
return content.toString();
};
export const parseFromYaml = (content: string) => {
return YAML.parse(content);
};
export const transformToYaml = (content: any) => {
return YAML.stringify(content);
};
export const updateFile = async (content: string, file: string) => {
await fs.writeFile(file, content);
};
export const fetchConfluence = async (relativeUrl: string, config: Config) => {
const baseUrl = config.getString('confluence.baseUrl');
const token = config.getString('confluence.token');
@@ -100,7 +87,6 @@ export const getAndWriteAttachments = async (
Authorization: `Bearer ${token}`,
},
});
// console.log(`${workspace}/${mkdocsDir}docs/img/${downloadTitle}`)
if (!res.ok) {
throw ResponseError.fromResponse(res);
} else if (res.body !== null) {
@@ -126,11 +112,11 @@ export const getAndWriteAttachments = async (
};
export const createConfluenceVariables = async (url: string) => {
let spacekey: string | null = null;
let title: string | null = null;
let titleWithSpaces: string | null = '';
let spacekey: string | undefined = undefined;
let title: string | undefined = undefined;
let titleWithSpaces: string | undefined = '';
const params = new URL(url);
if (url.includes('display')) {
if (params.pathname.split('/')[1] === 'display') {
spacekey = params.pathname.split('/')[2];
title = params.pathname.split('/')[3];
titleWithSpaces = title?.replace(/\+/g, ' ');
plugins/scaffolder-backend-module-confluence-to-markdowwn/src/index.ts
+1 -1
View File
@@ -15,7 +15,7 @@
*/
/**
* The confluence-to-markdowwn module for @backstage/plugin-scaffolder-backend.
* The confluence-to-markdown module for @backstage/plugin-scaffolder-backend.
*
* @packageDocumentation
*/
yarn.lock
+381 -336
View File
File diff suppressed because it is too large Load Diff