From 1797c5ce5dd660db633d22d4b2ad90b0ac431985 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 8 Sep 2021 13:29:35 +0200 Subject: [PATCH 1/6] catalog-backend: Drop support for deprecated location types Signed-off-by: Johan Haals --- .changeset/strange-buckets-design.md | 16 ++++++++++++++++ .../ingestion/processors/UrlReaderProcessor.ts | 18 +----------------- 2 files changed, 17 insertions(+), 17 deletions(-) create mode 100644 .changeset/strange-buckets-design.md diff --git a/.changeset/strange-buckets-design.md b/.changeset/strange-buckets-design.md new file mode 100644 index 0000000000..3cdf858174 --- /dev/null +++ b/.changeset/strange-buckets-design.md @@ -0,0 +1,16 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +This change drops support for deprecated location types which have all been replaced by the `url` type. +There has been a deprecation warning in place since the beginning of this year so most should already be migrated and received information at this point. + +The now removed location types are: + +``` +github +github/api +bitbucket/api +gitlab/api +azure/api +``` diff --git a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts index da7bee12d8..e3ec8c0480 100644 --- a/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/UrlReaderProcessor.ts @@ -26,16 +26,6 @@ import { CatalogProcessorParser, } from './types'; -// TODO(Rugvip): Added for backwards compatibility when moving to UrlReader, this -// can be removed in a bit -const deprecatedTypes = [ - 'github', - 'github/api', - 'bitbucket/api', - 'gitlab/api', - 'azure/api', -]; - type Options = { reader: UrlReader; logger: Logger; @@ -50,13 +40,7 @@ export class UrlReaderProcessor implements CatalogProcessor { emit: CatalogProcessorEmit, parser: CatalogProcessorParser, ): Promise { - if (deprecatedTypes.includes(location.type)) { - // TODO(Rugvip): Remove this warning a month or two into 2021, and remove support for the deprecated types. - this.options.logger.warn( - `Location '${location.target}' uses deprecated location type '${location.type}', use 'url' instead. ` + - 'Use "scripts/migrate-location-types.js" in the Backstage repo to migrate existing locations.', - ); - } else if (location.type !== 'url') { + if (location.type !== 'url') { return false; } From af05164b67eb33d09cccd5c1a89eeafe72aa6e49 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 8 Sep 2021 13:49:28 +0200 Subject: [PATCH 2/6] chore: fix tests Signed-off-by: Johan Haals --- plugins/catalog-backend/src/service/CatalogBuilder.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts index 579772dcce..87a26ed3aa 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.test.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.test.ts @@ -166,7 +166,7 @@ describe('CatalogBuilder', () => { const { entitiesCatalog, higherOrderOperation } = await builder.build(); await higherOrderOperation.addLocation({ - type: 'github', + type: 'url', target: 'https://github.com/a/b/x.yaml', }); const { entities } = await entitiesCatalog.entities(); From 6548b9e23f0244b2624b6b486f1b70e7747f5267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlio=20Zynger?= Date: Wed, 8 Sep 2021 13:20:56 +0200 Subject: [PATCH 3/6] Add code sample for ConfigApi access in frontend Per https://discord.com/channels/687207715902193673/687235481154617364/885085310101377034 Signed-off-by: Julio Zynger --- docs/conf/reading.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/conf/reading.md b/docs/conf/reading.md index 6ac10a8c8b..d492723e3d 100644 --- a/docs/conf/reading.md +++ b/docs/conf/reading.md @@ -112,7 +112,16 @@ example `getString`. These will throw an error if there is no value available. The [ConfigApi](../reference/utility-apis/Config.md) in the frontend is a [UtilityApi](../api/utility-apis.md). It's accessible as usual via the -`configApiRef` exported from `@backstage/core-plugin-api`. +`configApiRef` exported from `@backstage/core-plugin-api`: + +``` +import { useApi, configApiRef } from '@backstage/core-plugin-api'; +... +const MyReactComponent = (...) => { + const config = useApi(configApiRef); + ... +} +``` Depending on the config api in another API is slightly different though, as the `ConfigApi` implementation is supplied via the App itself and not instantiated From 24a70ae1eae56462bdeb77e0ff85937775280233 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Sep 2021 04:12:43 +0000 Subject: [PATCH 4/6] chore(deps-dev): bump @types/tar from 4.0.4 to 4.0.5 Bumps [@types/tar](https://github.com/DefinitelyTyped/DefinitelyTyped/tree/HEAD/types/tar) from 4.0.4 to 4.0.5. - [Release notes](https://github.com/DefinitelyTyped/DefinitelyTyped/releases) - [Commits](https://github.com/DefinitelyTyped/DefinitelyTyped/commits/HEAD/types/tar) --- updated-dependencies: - dependency-name: "@types/tar" dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yarn.lock b/yarn.lock index 5db9fd7589..e58e6fe0ac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7417,9 +7417,9 @@ integrity sha512-0VBprVqfgFD7Ehb2vd8Lh9TG3jP98gvr8rgehQqzztZNI7o8zS8Ad4jyZneKELphpuE212D8J70LnSNQSyO6bQ== "@types/tar@^4.0.3": - version "4.0.4" - resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.4.tgz#d680de60855e7778a51c672b755869a3b8d2889f" - integrity sha512-0Xv+xcmkTsOZdIF4yCnd7RkOOyfyqPaqJ7RZFKnwdxfDbkN3eAAE9sHl8zJFqBz4VhxolW9EErbjR1oyH7jK2A== + version "4.0.5" + resolved "https://registry.npmjs.org/@types/tar/-/tar-4.0.5.tgz#5f953f183e36a15c6ce3f336568f6051b7b183f3" + integrity sha512-cgwPhNEabHaZcYIy5xeMtux2EmYBitfqEceBUi2t5+ETy4dW6kswt6WX4+HqLeiiKOo42EXbGiDmVJ2x+vi37Q== dependencies: "@types/minipass" "*" "@types/node" "*" @@ -27589,9 +27589,9 @@ write-pkg@^4.0.0: write-json-file "^3.2.0" ws@7.4.5, ws@^7.4.6: - version "7.5.4" - resolved "https://registry.npmjs.org/ws/-/ws-7.5.4.tgz#56bfa20b167427e138a7795de68d134fe92e21f9" - integrity sha512-zP9z6GXm6zC27YtspwH99T3qTG7bBFv2VIkeHstMLrLlDJuzA7tQ5ls3OJ1hOGGCzTQPniNJoHXIAOS0Jljohg== + version "7.5.5" + resolved "https://registry.npmjs.org/ws/-/ws-7.5.5.tgz#8b4bc4af518cfabd0473ae4f99144287b33eb881" + integrity sha512-BAkMFcAzl8as1G/hArkxOxq3G7pjUqQ3gzYbLL0/5zNkph70e+lCoxBGnm6AW1+/aiNeV4fnKqZ8m4GZewmH2w== ws@^5.2.0: version "5.2.3" From f63efec0ece9aaf39d0afd1ef2d08d2beb9a248c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 9 Sep 2021 11:39:33 +0200 Subject: [PATCH 5/6] Revert "fix `subpath` filtering for `AzureDevOps` `readTree`" Signed-off-by: Patrik Oldsberg --- .changeset/early-seahorses-stare.md | 5 ---- .../src/reading/AzureUrlReader.test.ts | 15 ---------- .../src/reading/AzureUrlReader.ts | 5 ++-- .../reading/tree/ZipArchiveResponse.test.ts | 30 +++++++++++++++++++ .../src/reading/tree/ZipArchiveResponse.ts | 20 +++++++++++-- 5 files changed, 49 insertions(+), 26 deletions(-) delete mode 100644 .changeset/early-seahorses-stare.md diff --git a/.changeset/early-seahorses-stare.md b/.changeset/early-seahorses-stare.md deleted file mode 100644 index 963058a26f..0000000000 --- a/.changeset/early-seahorses-stare.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': patch ---- - -Fixing issue with `AzureUrlReader` that doesn't do `subpath` directories correctly diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index fc1f6a03f3..c7c8853929 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -220,21 +220,6 @@ describe('AzureUrlReader', () => { expect(indexMarkdownFile.toString()).toBe('# Test\n'); }); - it('returns the wanted files from an archive when a subpath is passed through', async () => { - const response = await processor.readTree( - 'https://dev.azure.com/organization/project/_git/repository?path=%2Fdocs', - ); - - expect(response.etag).toBe('123abc2'); - - const files = await response.files(); - - expect(files.length).toBe(1); - const indexMarkdownFile = await files[0].content(); - - expect(indexMarkdownFile.toString()).toBe('# Test\n'); - }); - it('creates a directory with the wanted files', async () => { const response = await processor.readTree( 'https://dev.azure.com/organization/project/_git/repository', diff --git a/packages/backend-common/src/reading/AzureUrlReader.ts b/packages/backend-common/src/reading/AzureUrlReader.ts index 2269393cbf..910e8e04a5 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.ts @@ -94,6 +94,8 @@ export class AzureUrlReader implements UrlReader { url: string, options?: ReadTreeOptions, ): Promise { + // TODO: Support filepath based reading tree feature like other providers + // Get latest commit SHA const commitsAzureResponse = await fetch( @@ -127,13 +129,10 @@ export class AzureUrlReader implements UrlReader { throw new Error(message); } - const { filepath } = parseGitUrl(url); - return await this.deps.treeResponseFactory.fromZipArchive({ stream: archiveAzureResponse.body as unknown as Readable, etag: commitSha, filter: options?.filter, - subpath: filepath, }); } diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index a54bac0c4a..f6f9a7845a 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -62,6 +62,36 @@ describe('ZipArchiveResponse', () => { ]); }); + it('should read files and strip root dir if requested', async () => { + const stream = fs.createReadStream('/test-archive-with-extra-root-dir.zip'); + + const res = new ZipArchiveResponse( + stream, + '', + '/tmp', + 'etag', + undefined, + true, + ); + const files = await res.files(); + + expect(files).toEqual([ + { + path: 'mkdocs.yml', + content: expect.any(Function), + }, + { + path: 'docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index e07cedde87..fb22f24b94 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -24,6 +24,7 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; +import { stripFirstDirectoryFromPath } from './util'; /** * Wraps a zip archive stream into a tree response reader. @@ -37,6 +38,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly workDir: string, public readonly etag: string, private readonly filter?: (path: string, info: { size: number }) => boolean, + private readonly stripFirstDirectory?: boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -66,8 +68,12 @@ export class ZipArchiveResponse implements ReadTreeResponse { } private shouldBeIncluded(entry: Entry): boolean { + const strippedPath = this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path; + if (this.subPath) { - if (!entry.path.startsWith(this.subPath)) { + if (!strippedPath.startsWith(this.subPath)) { return false; } } @@ -96,7 +102,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { files.push({ - path: this.getInnerPath(entry.path), + path: this.getInnerPath( + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, + ), content: () => entry.buffer(), }); } else { @@ -144,7 +154,11 @@ export class ZipArchiveResponse implements ReadTreeResponse { // Ignore directory entries since we handle that with the file entries // as a zip can have files with directories without directory entries if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath(entry.path); + const entryPath = this.getInnerPath( + this.stripFirstDirectory + ? stripFirstDirectoryFromPath(entry.path) + : entry.path, + ); const dirname = platformPath.dirname(entryPath); if (dirname) { await fs.mkdirp(platformPath.join(dir, dirname)); From f2ed25804b37c286872248aa1ce3dab533e08f01 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 9 Sep 2021 11:50:59 +0200 Subject: [PATCH 6/6] backend-common: keep ZipArchiveResponse removal of stripFirstDirectory Signed-off-by: Patrik Oldsberg --- .../reading/tree/ZipArchiveResponse.test.ts | 30 ------------------- .../src/reading/tree/ZipArchiveResponse.ts | 20 ++----------- 2 files changed, 3 insertions(+), 47 deletions(-) diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index f6f9a7845a..a54bac0c4a 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -62,36 +62,6 @@ describe('ZipArchiveResponse', () => { ]); }); - it('should read files and strip root dir if requested', async () => { - const stream = fs.createReadStream('/test-archive-with-extra-root-dir.zip'); - - const res = new ZipArchiveResponse( - stream, - '', - '/tmp', - 'etag', - undefined, - true, - ); - const files = await res.files(); - - expect(files).toEqual([ - { - path: 'mkdocs.yml', - content: expect.any(Function), - }, - { - path: 'docs/index.md', - content: expect.any(Function), - }, - ]); - const contents = await Promise.all(files.map(f => f.content())); - expect(contents.map(c => c.toString('utf8').trim())).toEqual([ - 'site_name: Test', - '# Test', - ]); - }); - it('should read files with filter', async () => { const stream = fs.createReadStream('/test-archive.zip'); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index fb22f24b94..e07cedde87 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -24,7 +24,6 @@ import { ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; -import { stripFirstDirectoryFromPath } from './util'; /** * Wraps a zip archive stream into a tree response reader. @@ -38,7 +37,6 @@ export class ZipArchiveResponse implements ReadTreeResponse { private readonly workDir: string, public readonly etag: string, private readonly filter?: (path: string, info: { size: number }) => boolean, - private readonly stripFirstDirectory?: boolean, ) { if (subPath) { if (!subPath.endsWith('/')) { @@ -68,12 +66,8 @@ export class ZipArchiveResponse implements ReadTreeResponse { } private shouldBeIncluded(entry: Entry): boolean { - const strippedPath = this.stripFirstDirectory - ? stripFirstDirectoryFromPath(entry.path) - : entry.path; - if (this.subPath) { - if (!strippedPath.startsWith(this.subPath)) { + if (!entry.path.startsWith(this.subPath)) { return false; } } @@ -102,11 +96,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { if (this.shouldBeIncluded(entry)) { files.push({ - path: this.getInnerPath( - this.stripFirstDirectory - ? stripFirstDirectoryFromPath(entry.path) - : entry.path, - ), + path: this.getInnerPath(entry.path), content: () => entry.buffer(), }); } else { @@ -154,11 +144,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { // Ignore directory entries since we handle that with the file entries // as a zip can have files with directories without directory entries if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath( - this.stripFirstDirectory - ? stripFirstDirectoryFromPath(entry.path) - : entry.path, - ); + const entryPath = this.getInnerPath(entry.path); const dirname = platformPath.dirname(entryPath); if (dirname) { await fs.mkdirp(platformPath.join(dir, dirname));