From 0d1c461780fad011bf48d46420f12b2be473e65f Mon Sep 17 00:00:00 2001 From: su-gupta Date: Tue, 22 Feb 2022 18:11:14 -0500 Subject: [PATCH 1/2] adding Confluence search functionality upstream v1 Signed-off-by: su-gupta --- contrib/search/README.md | 3 + contrib/search/confluence/ConfluenceCollator | 87 +++++++++++++++++++ .../confluence/ConfluenceResultListItem | 50 +++++++++++ contrib/search/confluence/README.md | 6 ++ 4 files changed, 146 insertions(+) create mode 100644 contrib/search/README.md create mode 100644 contrib/search/confluence/ConfluenceCollator create mode 100644 contrib/search/confluence/ConfluenceResultListItem create mode 100644 contrib/search/confluence/README.md diff --git a/contrib/search/README.md b/contrib/search/README.md new file mode 100644 index 0000000000..9a43a05c4c --- /dev/null +++ b/contrib/search/README.md @@ -0,0 +1,3 @@ +# Search + +Contributions/extensions to the Search plugin diff --git a/contrib/search/confluence/ConfluenceCollator b/contrib/search/confluence/ConfluenceCollator new file mode 100644 index 0000000000..5d6668b953 --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator @@ -0,0 +1,87 @@ +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch' + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + + const ConfluenceUrlBase = 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api' + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {} + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + 'Authorization': `Basic ${process.env.CONFLUENCE_TOKEN}` + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson + } + + async function getSpaces(): Promise { + const data = await getConfluenceData(`${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`); + let spacesList = [] + if (data["results"]) { + const results = data["results"]; + for (const result of results) { + spacesList.push(result["key"]) + } + } + return spacesList + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = [] + for (var space of spaces) { + let next = true + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}` + while (next) { + const data = await getConfluenceData(requestUrl) + if (data["results"]) { + const results = data["results"] + for (const result of results) { + documentsList.push(result["_links"]["self"]) + } + if (data["_links"]["next"]) { + requestUrl = data["_links"]["base"] + data["_links"]["next"] + } else { + next = false + } + } else { + break + } + } + } + return documentsList + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = [] + for (var documentUrl of documents) { + const data = await getConfluenceData(documentUrl + '?expand=body.storage') + if (data["status"] && data["status"]=="current") { + const documentMetaData = { + title: data["title"], + text: data["body"]["storage"]["value"], + location: data["_links"]["base"] + data["_links"]["webui"], + } + documentInfo.push(documentMetaData) + } + } + return documentInfo + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList + } +} diff --git a/contrib/search/confluence/ConfluenceResultListItem b/contrib/search/confluence/ConfluenceResultListItem new file mode 100644 index 0000000000..cd6d13bdea --- /dev/null +++ b/contrib/search/confluence/ConfluenceResultListItem @@ -0,0 +1,50 @@ +import React from 'react'; +import { Link } from '@backstage/core-components'; +import { IndexableDocument } from '@backstage/search-common'; +import { + Divider, + ListItem, + ListItemIcon, + ListItemText, +} from '@material-ui/core'; + +type Props = { + result: IndexableDocument; +}; + +export const ConfluenceResultListItem = ({ result }: Props) => { + // Remove html tags from document text before displaying + const chars = []; + let isTag = false; + for (const c of result.text.substring(0, 500)) { + if (c === "<") { + isTag = true; + continue; + } + if (c === ">") { + isTag = false; + chars.push(" ") + continue; + } + if (!isTag) { + chars.push(c); + } + } + const excerpt = chars.join("").substring(0, 80) + (result.text.length > 80 ? "..." : ""); + + return ( + + + + + + + + + + ); +}; diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md new file mode 100644 index 0000000000..5ebd275560 --- /dev/null +++ b/contrib/search/confluence/README.md @@ -0,0 +1,6 @@ +# Confluence + +These files help you add Confluence as a source to the Backstage Search plugin. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app as TypeScript files. +Then, update your packages/app/src/components/search/SearchPage.tsx and packages/backend/src/plugins/search.ts +to include the new Search source. From 1bd6e019abd3e4e7dd845ec575d97b73e9a135b0 Mon Sep 17 00:00:00 2001 From: su-gupta Date: Tue, 1 Mar 2022 12:15:18 -0500 Subject: [PATCH 2/2] change to md files Signed-off-by: su-gupta --- contrib/search/confluence/ConfluenceCollator | 87 ----------------- .../search/confluence/ConfluenceCollator.md | 95 +++++++++++++++++++ ...ltListItem => ConfluenceResultListItem.md} | 19 +++- contrib/search/confluence/README.md | 32 ++++++- 4 files changed, 138 insertions(+), 95 deletions(-) delete mode 100644 contrib/search/confluence/ConfluenceCollator create mode 100644 contrib/search/confluence/ConfluenceCollator.md rename contrib/search/confluence/{ConfluenceResultListItem => ConfluenceResultListItem.md} (72%) diff --git a/contrib/search/confluence/ConfluenceCollator b/contrib/search/confluence/ConfluenceCollator deleted file mode 100644 index 5d6668b953..0000000000 --- a/contrib/search/confluence/ConfluenceCollator +++ /dev/null @@ -1,87 +0,0 @@ -import { DocumentCollator } from '@backstage/search-common'; -import fetch from 'cross-fetch' - -export class ConfluenceCollator implements DocumentCollator { - public readonly type: string = 'confluence'; - - async execute() { - - const ConfluenceUrlBase = 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api' - - async function getConfluenceData(requestUrl: string) { - var emptyJson = {} - try { - const res = await fetch(requestUrl, { - method: 'get', - headers: { - 'Authorization': `Basic ${process.env.CONFLUENCE_TOKEN}` - }, - }); - if (res.ok) { - return await res.json(); - } - } catch (err) { - console.error(err); - } - return emptyJson - } - - async function getSpaces(): Promise { - const data = await getConfluenceData(`${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`); - let spacesList = [] - if (data["results"]) { - const results = data["results"]; - for (const result of results) { - spacesList.push(result["key"]) - } - } - return spacesList - } - - async function getDocumentsFromSpaces(spaces: string[]): Promise { - let documentsList = [] - for (var space of spaces) { - let next = true - let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}` - while (next) { - const data = await getConfluenceData(requestUrl) - if (data["results"]) { - const results = data["results"] - for (const result of results) { - documentsList.push(result["_links"]["self"]) - } - if (data["_links"]["next"]) { - requestUrl = data["_links"]["base"] + data["_links"]["next"] - } else { - next = false - } - } else { - break - } - } - } - return documentsList - } - - async function getDocumentInfo(documents: string[]) { - let documentInfo = [] - for (var documentUrl of documents) { - const data = await getConfluenceData(documentUrl + '?expand=body.storage') - if (data["status"] && data["status"]=="current") { - const documentMetaData = { - title: data["title"], - text: data["body"]["storage"]["value"], - location: data["_links"]["base"] + data["_links"]["webui"], - } - documentInfo.push(documentMetaData) - } - } - return documentInfo - } - - const spacesList = await getSpaces(); - const documentsList = await getDocumentsFromSpaces(spacesList); - const documentMetaDataList = await getDocumentInfo(documentsList); - return documentMetaDataList - } -} diff --git a/contrib/search/confluence/ConfluenceCollator.md b/contrib/search/confluence/ConfluenceCollator.md new file mode 100644 index 0000000000..f636ae88ea --- /dev/null +++ b/contrib/search/confluence/ConfluenceCollator.md @@ -0,0 +1,95 @@ +ConfluenceCollator.ts reference + +```ts +import { DocumentCollator } from '@backstage/search-common'; +import fetch from 'cross-fetch'; + +export class ConfluenceCollator implements DocumentCollator { + public readonly type: string = 'confluence'; + + async execute() { + const ConfluenceUrlBase = + 'https://{CONFLUENCE-ORG-NAME}.atlassian.net/wiki/rest/api'; + + async function getConfluenceData(requestUrl: string) { + var emptyJson = {}; + try { + const res = await fetch(requestUrl, { + method: 'get', + headers: { + Authorization: `Basic ${process.env.CONFLUENCE_TOKEN}`, + }, + }); + if (res.ok) { + return await res.json(); + } + } catch (err) { + console.error(err); + } + return emptyJson; + } + + async function getSpaces(): Promise { + const data = await getConfluenceData( + `${ConfluenceUrlBase}/space?&limit=1000&type=global&status=current`, + ); + let spacesList = []; + if (data['results']) { + const results = data['results']; + for (const result of results) { + spacesList.push(result['key']); + } + } + return spacesList; + } + + async function getDocumentsFromSpaces(spaces: string[]): Promise { + let documentsList = []; + for (var space of spaces) { + let next = true; + let requestUrl = `${ConfluenceUrlBase}/content?limit=1000&status=current&spaceKey=${space}`; + while (next) { + const data = await getConfluenceData(requestUrl); + if (data['results']) { + const results = data['results']; + for (const result of results) { + documentsList.push(result['_links']['self']); + } + if (data['_links']['next']) { + requestUrl = data['_links']['base'] + data['_links']['next']; + } else { + next = false; + } + } else { + break; + } + } + } + return documentsList; + } + + async function getDocumentInfo(documents: string[]) { + let documentInfo = []; + for (var documentUrl of documents) { + const data = await getConfluenceData( + documentUrl + '?expand=body.storage', + ); + if (data['status'] && data['status'] == 'current') { + const documentMetaData = { + title: data['title'], + text: data['body']['storage']['value'], + location: data['_links']['base'] + data['_links']['webui'], + }; + documentInfo.push(documentMetaData); + } + } + return documentInfo; + } + + const spacesList = await getSpaces(); + const documentsList = await getDocumentsFromSpaces(spacesList); + const documentMetaDataList = await getDocumentInfo(documentsList); + return documentMetaDataList; + } +} +``` diff --git a/contrib/search/confluence/ConfluenceResultListItem b/contrib/search/confluence/ConfluenceResultListItem.md similarity index 72% rename from contrib/search/confluence/ConfluenceResultListItem rename to contrib/search/confluence/ConfluenceResultListItem.md index cd6d13bdea..ba7c4f91a9 100644 --- a/contrib/search/confluence/ConfluenceResultListItem +++ b/contrib/search/confluence/ConfluenceResultListItem.md @@ -1,3 +1,6 @@ +ConfluenceResultListItem.tsx reference + +```tsx import React from 'react'; import { Link } from '@backstage/core-components'; import { IndexableDocument } from '@backstage/search-common'; @@ -17,26 +20,31 @@ export const ConfluenceResultListItem = ({ result }: Props) => { const chars = []; let isTag = false; for (const c of result.text.substring(0, 500)) { - if (c === "<") { + if (c === '<') { isTag = true; continue; } - if (c === ">") { + if (c === '>') { isTag = false; - chars.push(" ") + chars.push(' '); continue; } if (!isTag) { chars.push(c); } } - const excerpt = chars.join("").substring(0, 80) + (result.text.length > 80 ? "..." : ""); + const excerpt = + chars.join('').substring(0, 80) + (result.text.length > 80 ? '...' : ''); return ( - + { ); }; +``` diff --git a/contrib/search/confluence/README.md b/contrib/search/confluence/README.md index 5ebd275560..80beb9c169 100644 --- a/contrib/search/confluence/README.md +++ b/contrib/search/confluence/README.md @@ -1,6 +1,32 @@ # Confluence These files help you add Confluence as a source to the Backstage Search plugin. -To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app as TypeScript files. -Then, update your packages/app/src/components/search/SearchPage.tsx and packages/backend/src/plugins/search.ts -to include the new Search source. +To do so, add both files in this directory under the packages/backend/src/plugins/search/ pathway in your Backstage app. +Then, add the following code to your packages/app/src/components/search/SearchPage.tsx: + +```tsx +import { ConfluenceResultListItem } from './ConfluenceResultListItem'; +``` + +```tsx +case 'confluence': + return ( + + ); +``` + +and the following to packages/backend/src/plugins/search.ts: + +```ts +import { ConfluenceCollator } from './search/ConfluenceCollator'; +``` + +```ts +indexBuilder.addCollator({ + defaultRefreshIntervalSeconds: 600, + collator: new ConfluenceCollator(), +}); +```