diff --git a/.changeset/flat-pandas-tan.md b/.changeset/flat-pandas-tan.md new file mode 100644 index 0000000000..dfe7965bec --- /dev/null +++ b/.changeset/flat-pandas-tan.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +Added `createMockDirectory()` to help out with file system mocking in tests. diff --git a/.changeset/great-walls-brake.md b/.changeset/great-walls-brake.md new file mode 100644 index 0000000000..dd71c81a07 --- /dev/null +++ b/.changeset/great-walls-brake.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Properly close write stream when writing temporary archive for processing zip-based `.readTree()` responses. diff --git a/.changeset/short-terms-check.md b/.changeset/short-terms-check.md new file mode 100644 index 0000000000..69d85fcb9c --- /dev/null +++ b/.changeset/short-terms-check.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Removed `mock-fs` dev dependency. diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index e5d3adf013..312c662b5a 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -9,9 +9,10 @@ on: jobs: build: - runs-on: windows-2019 + runs-on: windows-2022 strategy: + fail-fast: false matrix: node-version: [18.x, 20.x] diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 344a9cdf25..a163311128 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -125,7 +125,6 @@ "@types/fs-extra": "^9.0.3", "@types/http-errors": "^2.0.0", "@types/minimist": "^1.2.0", - "@types/mock-fs": "^4.13.0", "@types/morgan": "^1.9.0", "@types/node-forge": "^1.3.0", "@types/pg": "^8.6.6", @@ -138,7 +137,6 @@ "aws-sdk-client-mock": "^2.0.0", "better-sqlite3": "^8.0.0", "http-errors": "^2.0.0", - "mock-fs": "^5.2.0", "msw": "^1.0.0", "mysql2": "^2.2.5", "recursive-readdir": "^2.2.2", diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 6b2f41a2bf..2b4bc84be7 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -23,12 +23,13 @@ import { AzureDevOpsCredentialLike, AzureIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import * as os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { getVoidLogger } from '../logging'; @@ -43,6 +44,8 @@ type AzureIntegrationConfigLike = Partial< const logger = getVoidLogger(); +const mockDir = createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -69,18 +72,8 @@ const urlReaderFactory = (azureIntegration: AzureIntegrationConfigLike) => { ); }; -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('AzureUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(mockDir.clear); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -270,7 +263,7 @@ describe('AzureUrlReader', () => { 'https://dev.azure.com/organization/project/_git/repository', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts index c00aeba124..dc182af619 100644 --- a/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts @@ -19,18 +19,21 @@ import { BitbucketCloudIntegration, readBitbucketCloudIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; +const mockDir = createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -49,18 +52,8 @@ const reader = new BitbucketCloudUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('BitbucketCloudUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(mockDir.clear); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -316,7 +309,7 @@ describe('BitbucketCloudUrlReader', () => { 'https://bitbucket.org/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -346,7 +339,7 @@ describe('BitbucketCloudUrlReader', () => { 'https://bitbucket.org/backstage/mock/src/master/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts index 9f231e2ba7..2870138535 100644 --- a/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts @@ -19,17 +19,20 @@ import { BitbucketServerIntegration, readBitbucketServerIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -46,19 +49,7 @@ const reader = new BitbucketServerUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('BitbucketServerUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const worker = setupServer(); setupRequestMockHandlers(worker); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 15e007d516..4258ac9d5b 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -19,12 +19,13 @@ import { BitbucketIntegration, readBitbucketIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; @@ -68,6 +69,10 @@ describe('BitbucketUrlReader.factory', () => { }); describe('BitbucketUrlReader', () => { + const mockDir = createMockDirectory({ mockOsTmpDir: true }); + + beforeEach(mockDir.clear); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -98,18 +103,6 @@ describe('BitbucketUrlReader', () => { { treeResponseFactory }, ); - const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const worker = setupServer(); setupRequestMockHandlers(worker); @@ -391,7 +384,7 @@ describe('BitbucketUrlReader', () => { 'https://bitbucket.org/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -436,7 +429,7 @@ describe('BitbucketUrlReader', () => { 'https://bitbucket.org/backstage/mock/src/master/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/GerritUrlReader.test.ts b/packages/backend-common/src/reading/GerritUrlReader.test.ts index 19c67ffd56..4a3b0a5de6 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.test.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.test.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { NotModifiedError, NotFoundError } from '@backstage/errors'; import { @@ -24,7 +27,6 @@ import { import { JsonObject } from '@backstage/types'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import mockFs from 'mock-fs'; import fs from 'fs-extra'; import path from 'path'; import { getVoidLogger } from '../logging'; @@ -33,6 +35,8 @@ import { DefaultReadTreeResponseFactory } from './tree'; import { GerritUrlReader } from './GerritUrlReader'; import getRawBody from 'raw-body'; +const mockDir = createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -89,6 +93,8 @@ describe.skip('GerritUrlReader', () => { const worker = setupServer(); setupRequestMockHandlers(worker); + beforeEach(mockDir.clear); + afterAll(() => { jest.clearAllMocks(); }); @@ -249,14 +255,13 @@ describe.skip('GerritUrlReader', () => { path.resolve(__dirname, '__fixtures__/gerrit/gerrit-master-docs.tar.gz'), ); - beforeEach(() => { - mockFs({ - '/tmp/': mockFs.directory(), - '/tmp/gerrit-clone-123abc/repo/mkdocs.yml': mkdocsContent, - '/tmp/gerrit-clone-123abc/repo/docs/first.md': mdContent, + beforeEach(async () => { + mockDir.setContent({ + 'repo/mkdocs.yml': mkdocsContent, + 'repo/docs/first.md': mdContent, }); const spy = jest.spyOn(fs, 'mkdtemp'); - spy.mockImplementation(() => '/tmp/gerrit-clone-123abc'); + spy.mockImplementation(() => mockDir.path); worker.use( rest.get( @@ -289,7 +294,6 @@ describe.skip('GerritUrlReader', () => { }); afterEach(() => { - mockFs.restore(); jest.clearAllMocks(); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 499f97d6ec..5e6d30f519 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -20,12 +20,13 @@ import { GithubIntegration, readGithubIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; import { @@ -37,6 +38,8 @@ import { } from './GithubUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +const mockDir = createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -69,21 +72,11 @@ const gheProcessor = new GithubUrlReader( { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('GithubUrlReader', () => { const worker = setupServer(); setupRequestMockHandlers(worker); - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(mockDir.clear); beforeEach(() => { jest.clearAllMocks(); @@ -420,7 +413,7 @@ describe('GithubUrlReader', () => { 'https://github.com/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -501,7 +494,7 @@ describe('GithubUrlReader', () => { 'https://github.com/backstage/mock/tree/main/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 5af4faf7e5..3c065e8b81 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -15,12 +15,13 @@ */ import { ConfigReader } from '@backstage/config'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + createMockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; @@ -33,6 +34,8 @@ import { const logger = getVoidLogger(); +const mockDir = createMockDirectory({ mockOsTmpDir: true }); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -65,18 +68,8 @@ const hostedGitlabProcessor = new GitlabUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('GitlabUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(mockDir.clear); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -400,7 +393,7 @@ describe('GitlabUrlReader', () => { 'https://gitlab.com/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -457,7 +450,7 @@ describe('GitlabUrlReader', () => { 'https://gitlab.com/backstage/mock/tree/main/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index a5728a6169..0a27bdf09c 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -15,41 +15,54 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; -import path, { resolve as resolvePath } from 'path'; +import path from 'path'; import { FromReadableArrayOptions } from '../types'; import { ReadableArrayResponse } from './ReadableArrayResponse'; +import { createMockDirectory } from '@backstage/backend-test-utils'; -const path1 = '/file1.yaml'; +const name1 = 'file1.yaml'; const file1 = fs.readFileSync( path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object.yaml'), ); -const path2 = '/file2.yaml'; +const name2 = 'file2.yaml'; const file2 = fs.readFileSync( path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object2.yaml'), ); describe('ReadableArrayResponse', () => { + const sourceDir = createMockDirectory(); + const targetDir = createMockDirectory(); + beforeEach(() => { - mockFs({ - [path1]: file1, - [path2]: file2, - '/tmp': mockFs.directory(), + sourceDir.setContent({ + [name1]: file1, + [name2]: file2, }); + targetDir.clear(); }); + const openStreams = new Array(); + function createReadStream(filePath: string) { + const stream = fs.createReadStream(filePath); + openStreams.push(stream); + return stream; + } afterEach(() => { - mockFs.restore(); + openStreams.forEach(stream => stream.destroy()); + openStreams.length = 0; }); + const path1 = sourceDir.resolve(name1); + const path2 = sourceDir.resolve(name2); + it('should read files', async () => { const arr: FromReadableArrayOptions = [ - { data: fs.createReadStream(path1), path: path1 }, - { data: fs.createReadStream(path2), path: path2 }, + { data: createReadStream(path1), path: path1 }, + { data: createReadStream(path2), path: path2 }, ]; - const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); + const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -57,26 +70,21 @@ describe('ReadableArrayResponse', () => { { path: path2, content: expect.any(Function) }, ]); const contents = await Promise.all(files.map(f => f.content())); - expect(contents.map(c => c.toString('utf8').trim())).toEqual([ - 'site_name: Test', - 'site_name: Test2', - ]); + expect(contents).toEqual([file1, file2]); }); it('should extract entire archive into directory', async () => { const arr: FromReadableArrayOptions = [ - { data: fs.createReadStream(path1), path: path1 }, - { data: fs.createReadStream(path2), path: path2 }, + { data: createReadStream(path1), path: path1 }, + { data: createReadStream(path2), path: path2 }, ]; - const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); + const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); const dir = await res.dir(); - await expect( - fs.readFile(resolvePath(dir, 'file1.yaml'), 'utf8'), - ).resolves.toMatch(/site_name: Test/); - await expect( - fs.readFile(resolvePath(dir, 'file2.yaml'), 'utf8'), - ).resolves.toMatch(/site_name: Test2/); + expect(targetDir.content({ path: dir })).toEqual({ + [name1]: file1.toString('utf8'), + [name2]: file2.toString('utf8'), + }); }); }); diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts index 1765a41156..f69fb2e0e1 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts @@ -15,30 +15,31 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { resolve as resolvePath } from 'path'; import { TarArchiveResponse } from './TarArchiveResponse'; +import { createMockDirectory } from '@backstage/backend-test-utils'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'), ); describe('TarArchiveResponse', () => { - beforeEach(() => { - mockFs({ - '/test-archive.tar.gz': archiveData, - '/tmp': mockFs.directory(), - }); - }); + const sourceDir = createMockDirectory(); + const targetDir = createMockDirectory(); - afterEach(() => { - mockFs.restore(); + beforeAll(() => { + sourceDir.setContent({ 'test-archive.tar.gz': archiveData }); + }); + beforeEach(() => { + targetDir.clear(); }); it('should read files', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -61,10 +62,16 @@ describe('TarArchiveResponse', () => { }); it('should read files with filter', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -80,16 +87,18 @@ describe('TarArchiveResponse', () => { }); it('should read as archive and files', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new TarArchiveResponse(buffer, '', '/tmp', 'etag'); + const res2 = new TarArchiveResponse(buffer, '', targetDir.path, 'etag'); const files = await res2.files(); expect(files).toEqual([ @@ -112,9 +121,11 @@ describe('TarArchiveResponse', () => { }); it('should extract entire archive into directory', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const dir = await res.dir(); await expect( fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'), @@ -125,67 +136,91 @@ describe('TarArchiveResponse', () => { }); it('should extract archive into directory with a subpath', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, 'docs', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, 'docs', targetDir.path, 'etag'); const dir = await res.dir(); - expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); - await expect( - fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), - ).resolves.toBe('# Test\n'); + expect(targetDir.content({ path: dir })).toEqual({ + 'index.md': '# Test\n', + }); }); it('should extract archive into directory with a subpath and filter', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); - - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), ); - const dir = await res.dir({ targetDir: '/tmp' }); - expect(dir).toBe('/tmp'); - await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( - true, + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); - await expect( - fs.pathExists(resolvePath(dir, 'docs/index.md')), - ).resolves.toBe(false); + + targetDir.addContent({ sub: {} }); + const dir = await res.dir({ targetDir: targetDir.resolve('sub') }); + + expect(dir).toBe(targetDir.resolve('sub')); + expect(targetDir.content()).toEqual({ + sub: { + 'mkdocs.yml': 'site_name: Test\n', + }, + }); }); - it('should leave temporary directories in place in the case of an error', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + it('should clean up temporary directories in place in the case of an error', async () => { + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', () => { - throw new Error('NOPE'); - }); + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + () => { + throw new Error('NOPE'); + }, + ); - const tmpDir = await fs.mkdtemp('/tmp/test'); - // selects the wrong overload by default - const mkdtemp = jest.spyOn(fs, 'mkdtemp') as unknown as jest.SpyInstance< - Promise, - [] - >; - mkdtemp.mockResolvedValue(tmpDir); + targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); + const mkdtemp = jest + .spyOn(fs, 'mkdtemp') + .mockImplementation(async () => sub); + + await expect(fs.pathExists(sub)).resolves.toBe(true); await expect(res.dir()).rejects.toThrow('NOPE'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(false); + await expect(fs.pathExists(sub)).resolves.toBe(false); mkdtemp.mockRestore(); }); it('should leave directory in place if provided in the case of an error', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', () => { - throw new Error('NOPE'); - }); + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + () => { + throw new Error('NOPE'); + }, + ); - const tmpDir = await fs.mkdtemp('/tmp/test'); + targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); - await expect(res.dir({ targetDir: tmpDir })).rejects.toThrow('NOPE'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); + await expect(fs.pathExists(sub)).resolves.toBe(true); + await expect(res.dir({ targetDir: sub })).rejects.toThrow('NOPE'); + await expect(fs.pathExists(sub)).resolves.toBe(true); }); }); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index 0b010565dd..ccf53e7cff 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -15,11 +15,11 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { Readable } from 'stream'; import { create as createArchive } from 'archiver'; import { resolve as resolvePath } from 'path'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { createMockDirectory } from '@backstage/backend-test-utils'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.zip'), @@ -35,24 +35,36 @@ const archiveWithMaliciousEntry = fs.readFileSync( ); describe('ZipArchiveResponse', () => { - beforeEach(() => { - mockFs({ - '/test-archive.zip': archiveData, - '/test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, - '/test-archive-corrupted.zip': archiveDataCorrupted, - '/test-archive-malicious.zip': archiveWithMaliciousEntry, - '/tmp': mockFs.directory(), + const sourceDir = createMockDirectory(); + const targetDir = createMockDirectory(); + + beforeAll(() => { + sourceDir.setContent({ + 'test-archive.zip': archiveData, + 'test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, + 'test-archive-corrupted.zip': archiveDataCorrupted, + 'test-archive-malicious.zip': archiveWithMaliciousEntry, }); }); + beforeEach(() => { + targetDir.clear(); + }); + const openStreams = new Array(); + function createReadStream(filePath: string) { + const stream = fs.createReadStream(filePath); + openStreams.push(stream); + return stream; + } afterEach(() => { - mockFs.restore(); + openStreams.forEach(stream => stream.destroy()); + openStreams.length = 0; }); it('should read files', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -76,10 +88,14 @@ describe('ZipArchiveResponse', () => { }); it('should read files with filter', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -95,16 +111,16 @@ describe('ZipArchiveResponse', () => { }); it('should read as archive and files', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new ZipArchiveResponse(buffer, '', '/tmp', 'etag'); + const res2 = new ZipArchiveResponse(buffer, '', targetDir.path, 'etag'); const files = await res2.files(); expect(files).toEqual([ @@ -127,9 +143,9 @@ describe('ZipArchiveResponse', () => { }); it('should extract entire archive into directory', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const dir = await res.dir(); await expect( @@ -141,38 +157,45 @@ describe('ZipArchiveResponse', () => { }); it('should extract archive into directory with a subpath', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); + + const res = new ZipArchiveResponse(stream, 'docs/', targetDir.path, 'etag'); - const res = new ZipArchiveResponse(stream, 'docs/', '/tmp', 'etag'); const dir = await res.dir(); - expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); - await expect( - fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), - ).resolves.toBe('# Test\n'); + expect(targetDir.content({ path: dir })).toEqual({ + 'index.md': '# Test\n', + }); }); it('should extract archive into directory with a subpath and filter', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); - const dir = await res.dir({ targetDir: '/tmp' }); - expect(dir).toBe('/tmp'); - await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( - true, - ); - await expect( - fs.pathExists(resolvePath(dir, 'docs/index.md')), - ).resolves.toBe(false); + targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); + const dir = await res.dir({ targetDir: sub }); + + expect(dir).toBe(sub); + + expect(targetDir.content()).toEqual({ + sub: { + 'mkdocs.yml': 'site_name: Test\n', + }, + }); }); it('should extract a large archive', async () => { const fileCount = 10; const fileSize = 1000 * 1000; const filePath = await new Promise((resolve, reject) => { - const outFile = '/large-archive.zip'; + const outFile = targetDir.resolve('large-archive.zip'); const archive = createArchive('zip'); archive.on('error', reject); @@ -193,14 +216,16 @@ describe('ZipArchiveResponse', () => { archive.finalize(); }); - const stream = fs.createReadStream(filePath); + const stream = createReadStream(filePath); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); + + targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); const dir = await res.dir({ - targetDir: '/out', + targetDir: sub, }); - expect(dir).toBe('/out'); const files = await fs.readdir(dir); expect(files).toHaveLength(fileCount); @@ -211,9 +236,11 @@ describe('ZipArchiveResponse', () => { }); it('should throw on invalid archive', async () => { - const stream = fs.createReadStream('/test-archive-corrupted.zip'); + const stream = createReadStream( + sourceDir.resolve('test-archive-corrupted.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const filesPromise = res.files(); await expect(filesPromise).rejects.toThrow( @@ -222,18 +249,22 @@ describe('ZipArchiveResponse', () => { }); it('should throw on entries with a path outside the destination dir', async () => { - const stream = fs.createReadStream('/test-archive-malicious.zip'); + const stream = createReadStream( + sourceDir.resolve('test-archive-malicious.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); await expect(res.files()).rejects.toThrow( 'invalid relative path: ../side.txt', ); }); it('should throw on entries that attempt to write outside destination dir', async () => { - const stream = fs.createReadStream('/test-archive-malicious.zip'); + const stream = createReadStream( + sourceDir.resolve('test-archive-malicious.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); await expect(res.dir()).rejects.toThrow( 'invalid relative path: ../side.txt', ); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 1e55150310..a191163517 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -93,9 +93,13 @@ export class ZipArchiveResponse implements ReadTreeResponse { return new Promise((resolve, reject) => { writeStream.on('error', reject); - writeStream.on('finish', () => - resolve({ fileName: tmpFile, cleanup: () => fs.remove(tmpFile) }), - ); + writeStream.on('finish', () => { + writeStream.end(); + resolve({ + fileName: tmpFile, + cleanup: () => fs.rm(tmpDir, { recursive: true }), + }); + }); stream.pipe(writeStream); }); } @@ -152,7 +156,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { }); }); - temporary.cleanup(); + await temporary.cleanup(); return files; } @@ -175,7 +179,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { archive.finalize(); - temporary.cleanup(); + await temporary.cleanup(); return archive; } @@ -204,7 +208,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { }); }); - temporary.cleanup(); + await temporary.cleanup(); return dir; } diff --git a/packages/backend-common/src/util/DockerContainerRunner.test.ts b/packages/backend-common/src/util/DockerContainerRunner.test.ts index 3208679d1f..737bf3d178 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.test.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.test.ts @@ -14,27 +14,24 @@ * limitations under the License. */ +import fs from 'fs-extra'; import Docker from 'dockerode'; -import mockFs from 'mock-fs'; -import os from 'os'; -import path from 'path'; import Stream, { PassThrough } from 'stream'; import { ContainerRunner } from './ContainerRunner'; import { DockerContainerRunner, UserOptions } from './DockerContainerRunner'; +import { createMockDirectory } from '@backstage/backend-test-utils'; const mockDocker = new Docker() as jest.Mocked; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; describe('DockerContainerRunner', () => { let containerTaskApi: ContainerRunner; + const inputDir = createMockDirectory(); + const outputDir = createMockDirectory(); + beforeEach(() => { - mockFs({ - [rootDir]: { - input: mockFs.directory(), - output: mockFs.directory(), - }, - }); + inputDir.clear(); + outputDir.clear(); jest.spyOn(mockDocker, 'pull').mockImplementation((async ( _image: string, @@ -59,16 +56,15 @@ describe('DockerContainerRunner', () => { afterEach(() => { jest.clearAllMocks(); - mockFs.restore(); }); const imageName = 'dockerOrg/image'; const args = ['bash', '-c', 'echo test']; const mountDirs = { - [path.join(rootDir, 'input')]: '/input', - [path.join(rootDir, 'output')]: '/output', + [inputDir.path]: '/input', + [outputDir.path]: '/output', }; - const workingDir = path.join(rootDir, 'input'); + const workingDir = inputDir.path; const envVars = { HOME: '/tmp', LOG_LEVEL: 'debug' }; const envVarsArray = ['HOME=/tmp', 'LOG_LEVEL=debug']; @@ -117,8 +113,8 @@ describe('DockerContainerRunner', () => { HostConfig: { AutoRemove: true, Binds: expect.arrayContaining([ - `${path.join(rootDir, 'input')}:/input`, - `${path.join(rootDir, 'output')}:/output`, + `${await fs.realpath(inputDir.path)}:/input`, + `${await fs.realpath(outputDir.path)}:/output`, ]), }, Volumes: { diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 4a27ac21a9..a477f70298 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -4,6 +4,7 @@ ```ts /// +/// import { Backend } from '@backstage/backend-app-api'; import { BackendFeature } from '@backstage/backend-plugin-api'; @@ -30,9 +31,44 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; import { TokenManagerService } from '@backstage/backend-plugin-api'; import { UrlReaderService } from '@backstage/backend-plugin-api'; +// @public +export function createMockDirectory( + options?: MockDirectoryOptions, +): MockDirectory; + // @public (undocumented) export function isDockerDisabledForTests(): boolean; +// @public +export interface MockDirectory { + addContent(root: MockDirectoryContent): void; + clear(): void; + content( + options?: MockDirectoryContentOptions, + ): MockDirectoryContent | undefined; + readonly path: string; + remove(): void; + resolve(...paths: string[]): string; + setContent(root: MockDirectoryContent): void; +} + +// @public +export type MockDirectoryContent = { + [name in string]: MockDirectoryContent | string | Buffer; +}; + +// @public +export interface MockDirectoryContentOptions { + path?: string; + shouldReadAsText?: boolean | ((path: string, buffer: Buffer) => boolean); +} + +// @public +export interface MockDirectoryOptions { + content?: MockDirectoryContent; + mockOsTmpDir?: boolean; +} + // @public (undocumented) export namespace mockServices { // (undocumented) diff --git a/packages/backend-test-utils/package.json b/packages/backend-test-utils/package.json index 04be07ed33..691f0a921b 100644 --- a/packages/backend-test-utils/package.json +++ b/packages/backend-test-utils/package.json @@ -46,15 +46,18 @@ "@backstage/backend-common": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", + "@backstage/errors": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/types": "workspace:^", "better-sqlite3": "^8.0.0", "express": "^4.17.1", + "fs-extra": "^10.0.1", "knex": "^2.0.0", "msw": "^1.0.0", "mysql2": "^2.2.5", "pg": "^8.3.0", "testcontainers": "^8.1.2", + "textextensions": "^5.16.0", "uuid": "^8.0.0" }, "peerDependencies": { diff --git a/packages/backend-test-utils/src/filesystem/MockDirectory.test.ts b/packages/backend-test-utils/src/filesystem/MockDirectory.test.ts new file mode 100644 index 0000000000..9e3efdaec0 --- /dev/null +++ b/packages/backend-test-utils/src/filesystem/MockDirectory.test.ts @@ -0,0 +1,293 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import os from 'os'; +import { join as joinPath, relative as relativePath } from 'path'; +import { createMockDirectory, MockDirectory } from './MockDirectory'; + +describe('createMockDirectory', () => { + const mockDir = createMockDirectory(); + + beforeEach(mockDir.clear); + + it('should resolve paths', () => { + expect(mockDir.path).toEqual(expect.any(String)); + expect(relativePath(mockDir.path, mockDir.resolve('a'))).toBe('a'); + expect(relativePath(mockDir.path, mockDir.resolve('a/b/c'))).toBe( + joinPath('a', 'b', 'c'), + ); + }); + + it('should remove itself', async () => { + await expect(fs.pathExists(mockDir.path)).resolves.toBe(true); + mockDir.remove(); + await expect(fs.pathExists(mockDir.path)).resolves.toBe(false); + }); + + it('should populate a directory with text files', () => { + mockDir.setContent({ + 'a.txt': 'a', + 'a/b.txt': 'b', + 'a/b/c.txt': 'c', + 'a/b/d.txt': 'd', + }); + + expect(mockDir.content()).toEqual({ + 'a.txt': 'a', + a: { + 'b.txt': 'b', + b: { + 'c.txt': 'c', + 'd.txt': 'd', + }, + }, + }); + }); + + it('should mix text and binary files', () => { + mockDir.setContent({ + 'a.txt': 'a', + 'a/b.txt': 'b', + 'a/b/c.bin': Buffer.from([0xc]), + 'a/b/d.bin': Buffer.from([0xd]), + }); + + expect(mockDir.content()).toEqual({ + 'a.txt': 'a', + a: { + 'b.txt': 'b', + b: { + 'c.bin': Buffer.from([0xc]), + 'd.bin': Buffer.from([0xd]), + }, + }, + }); + }); + + it('should be able to add content', () => { + mockDir.setContent({ + 'a.txt': 'a', + b: {}, + }); + + expect(mockDir.content()).toEqual({ + 'a.txt': 'a', + b: {}, + }); + + mockDir.addContent({ + 'b.txt': 'b', + b: { + 'c.txt': 'c', + }, + }); + + expect(mockDir.content()).toEqual({ + 'a.txt': 'a', + 'b.txt': 'b', + b: { + 'c.txt': 'c', + }, + }); + }); + + it('should replace existing files', () => { + mockDir.setContent({ + 'a.txt': 'a', + }); + + mockDir.addContent({ + 'a.txt': 'a2', + }); + + expect(mockDir.content()).toEqual({ + 'a.txt': 'a2', + }); + }); + + it('should read content from sub dirs', () => { + mockDir.setContent({ + 'a.txt': 'a', + 'b/b.txt': 'b', + 'b/c/c.txt': 'c', + }); + + const expected = { + 'a.txt': 'a', + b: { + 'b.txt': 'b', + c: { + 'c.txt': 'c', + }, + }, + }; + + expect(mockDir.content()).toEqual(expected); + expect(mockDir.content({ path: mockDir.path })).toEqual(expected); + expect(mockDir.content({ path: mockDir.resolve('.') })).toEqual(expected); + expect(mockDir.content({ path: 'b' })).toEqual(expected.b); + expect(mockDir.content({ path: './b' })).toEqual(expected.b); + expect(mockDir.content({ path: mockDir.resolve('b') })).toEqual(expected.b); + expect(mockDir.content({ path: 'b/c' })).toEqual(expected.b.c); + expect(mockDir.content({ path: './b/c' })).toEqual(expected.b.c); + expect(mockDir.content({ path: mockDir.resolve('b/c') })).toEqual( + expected.b.c, + ); + expect(mockDir.content({ path: mockDir.resolve('b', 'c') })).toEqual( + expected.b.c, + ); + }); + + it('should allow text reading to be configured', () => { + const text = 'a'; + const binary = Buffer.from('a', 'utf8'); + + mockDir.setContent({ + a: binary, + 'a.txt': text, + 'a.bin': binary, + }); + + expect(mockDir.content()).toEqual({ + a: binary, + 'a.txt': text, + 'a.bin': binary, + }); + + expect(mockDir.content({ shouldReadAsText: false })).toEqual({ + a: binary, + 'a.txt': binary, + 'a.bin': binary, + }); + + expect(mockDir.content({ shouldReadAsText: true })).toEqual({ + a: text, + 'a.txt': text, + 'a.bin': text, + }); + + expect( + mockDir.content({ shouldReadAsText: path => path.length > 3 }), + ).toEqual({ + a: binary, + 'a.txt': text, + 'a.bin': text, + }); + }); + + it('should provide a posix path to shouldReadAsText', () => { + const shouldReadAsText = jest.fn().mockReturnValue(true); + + mockDir.setContent({ 'a/b/c': 'c' }); + + expect(mockDir.content({ shouldReadAsText })).toEqual({ + a: { b: { c: 'c' } }, + }); + expect(shouldReadAsText).toHaveBeenCalledWith( + 'a/b/c', + Buffer.from('c', 'utf8'), + ); + }); + + it('should not override directories', () => { + mockDir.setContent({ + 'a.txt': 'a', + b: {}, + }); + + expect(() => + mockDir.addContent({ + 'a.txt': {}, + }), + ).toThrow('EEXIST'); + + expect(() => + mockDir.addContent({ + b: 'b', + }), + ).toThrow('EISDIR'); + }); + + it('examples should work', () => { + mockDir.setContent({ + 'test.txt': 'content', + 'sub-dir': { + 'file.txt': 'content', + 'nested-dir/file.txt': 'content', + }, + 'empty-dir': {}, + 'binary-file': Buffer.from([0, 1, 2]), + }); + + mockDir.addContent({ + 'test.txt': 'content', + 'sub-dir': { + 'file.txt': 'content', + 'nested-dir/file.txt': 'content', + }, + 'empty-dir': {}, + 'binary-file': Buffer.from([0, 1, 2]), + }); + + expect(mockDir.content()).toEqual({ + 'test.txt': 'content', + 'sub-dir': { + 'file.txt': 'content', + 'nested-dir': { + 'file.txt': 'content', + }, + }, + 'empty-dir': {}, + 'binary-file': Buffer.from([0, 1, 2]), + }); + }); + + it('should reject non-child paths', () => { + const path = mockDir.resolve('/root/a.txt'); + expect(() => mockDir.setContent({ '/root/a.txt': 'a' })).toThrow( + `Provided path must resolve to a child path of the mock directory, got '${path}'`, + ); + expect(() => mockDir.addContent({ '/root/a.txt': 'a' })).toThrow( + `Provided path must resolve to a child path of the mock directory, got '${path}'`, + ); + expect(() => mockDir.content({ path: '/root/a.txt' })).toThrow( + `Provided path must resolve to a child path of the mock directory, got '${path}'`, + ); + }); + + describe('tmpdir mock', () => { + let tmpDirMock: MockDirectory; + + describe('inner', () => { + tmpDirMock = createMockDirectory({ mockOsTmpDir: true }); + + it('should mock os.tmpdir()', () => { + expect(os.tmpdir()).toBe(tmpDirMock.path); + }); + + it('should refuce to mock os.tmpdir() again', () => { + expect(() => createMockDirectory({ mockOsTmpDir: true })).toThrow( + 'Cannot mock os.tmpdir() when it has already been mocked', + ); + }); + }); + + it('should restore os.tmpdir()', () => { + expect(os.tmpdir()).not.toBe(tmpDirMock.path); + }); + }); +}); diff --git a/packages/backend-test-utils/src/filesystem/MockDirectory.ts b/packages/backend-test-utils/src/filesystem/MockDirectory.ts new file mode 100644 index 0000000000..20345e4ff9 --- /dev/null +++ b/packages/backend-test-utils/src/filesystem/MockDirectory.ts @@ -0,0 +1,391 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import { isChildPath } from '@backstage/backend-common'; +import fs from 'fs-extra'; +import textextensions from 'textextensions'; +import { + dirname, + extname, + join as joinPath, + resolve as resolvePath, + relative as relativePath, + win32, + posix, +} from 'path'; + +const tmpdirMarker = Symbol('os-tmpdir-mock'); + +/** + * The content of a mock directory represented by a nested object structure. + * + * @remarks + * + * When used as input, the keys may contain forward slashes to indicate nested directories. + * Then returned as output, each directory will always be represented as a separate object. + * + * @example + * ```ts + * { + * 'test.txt': 'content', + * 'sub-dir': { + * 'file.txt': 'content', + * 'nested-dir/file.txt': 'content', + * }, + * 'empty-dir': {}, + * 'binary-file': Buffer.from([0, 1, 2]), + * } + * ``` + * + * @public + */ +export type MockDirectoryContent = { + [name in string]: MockDirectoryContent | string | Buffer; +}; + +/** + * Options for {@link MockDirectory.content}. + * + * @public + */ +export interface MockDirectoryContentOptions { + /** + * The path to read content from. Defaults to the root of the mock directory. + * + * An absolute path can also be provided, as long as it is a child path of the mock directory. + */ + path?: string; + + /** + * Whether or not to return files as text rather than buffers. + * + * Defaults to checking the file extension against a list of known text extensions. + */ + shouldReadAsText?: boolean | ((path: string, buffer: Buffer) => boolean); +} + +/** + * A utility for creating a mock directory that is automatically cleaned up. + * + * @public + */ +export interface MockDirectory { + /** + * The path to the root of the mock directory + */ + readonly path: string; + + /** + * Resolves a path relative to the root of the mock directory. + */ + resolve(...paths: string[]): string; + + /** + * Sets the content of the mock directory. This will remove any existing content. + * + * @example + * ```ts + * mockDir.setContent({ + * 'test.txt': 'content', + * 'sub-dir': { + * 'file.txt': 'content', + * 'nested-dir/file.txt': 'content', + * }, + * 'empty-dir': {}, + * 'binary-file': Buffer.from([0, 1, 2]), + * }); + * ``` + */ + setContent(root: MockDirectoryContent): void; + + /** + * Adds content of the mock directory. This will overwrite existing files. + * + * @example + * ```ts + * mockDir.addContent({ + * 'test.txt': 'content', + * 'sub-dir': { + * 'file.txt': 'content', + * 'nested-dir/file.txt': 'content', + * }, + * 'empty-dir': {}, + * 'binary-file': Buffer.from([0, 1, 2]), + * }); + * ``` + */ + addContent(root: MockDirectoryContent): void; + + /** + * Reads the content of the mock directory. + * + * @remarks + * + * Text files will be returned as strings, while binary files will be returned as buffers. + * By default the file extension is used to determine whether a file should be read as text. + * + * @example + * ```ts + * expect(mockDir.content()).toEqual({ + * 'test.txt': 'content', + * 'sub-dir': { + * 'file.txt': 'content', + * 'nested-dir': { + * 'file.txt': 'content', + * }, + * }, + * 'empty-dir': {}, + * 'binary-file': Buffer.from([0, 1, 2]), + * }); + * ``` + */ + content( + options?: MockDirectoryContentOptions, + ): MockDirectoryContent | undefined; + + /** + * Clears the content of the mock directory, ensuring that the directory itself exists. + */ + clear(): void; + + /** + * Removes the mock directory and all its contents. + */ + remove(): void; +} + +/** @internal */ +type MockEntry = + | { + type: 'file'; + path: string; + content: Buffer; + } + | { + type: 'dir'; + path: string; + }; + +/** @internal */ +class MockDirectoryImpl { + readonly #root: string; + + constructor(root: string) { + this.#root = root; + } + + get path(): string { + return this.#root; + } + + resolve(...paths: string[]): string { + return resolvePath(this.#root, ...paths); + } + + setContent(root: MockDirectoryContent): void { + this.remove(); + + return this.addContent(root); + } + + addContent(root: MockDirectoryContent): void { + const entries = this.#transformInput(root); + + for (const entry of entries) { + const fullPath = resolvePath(this.#root, entry.path); + if (!isChildPath(this.#root, fullPath)) { + throw new Error( + `Provided path must resolve to a child path of the mock directory, got '${fullPath}'`, + ); + } + + if (entry.type === 'dir') { + fs.ensureDirSync(fullPath, { mode: 0o777 }); + } else if (entry.type === 'file') { + fs.ensureDirSync(dirname(fullPath), { mode: 0o777 }); + fs.writeFileSync(fullPath, entry.content, { mode: 0o666 }); + } + } + } + + content( + options?: MockDirectoryContentOptions, + ): MockDirectoryContent | undefined { + const shouldReadAsText = + (typeof options?.shouldReadAsText === 'boolean' + ? () => options?.shouldReadAsText + : options?.shouldReadAsText) ?? + ((path: string) => textextensions.includes(extname(path).slice(1))); + + const root = resolvePath(this.#root, options?.path ?? ''); + if (!isChildPath(this.#root, root)) { + throw new Error( + `Provided path must resolve to a child path of the mock directory, got '${root}'`, + ); + } + + function read(path: string): MockDirectoryContent | undefined { + if (!fs.pathExistsSync(path)) { + return undefined; + } + + const entries = fs.readdirSync(path, { withFileTypes: true }); + return Object.fromEntries( + entries.map(entry => { + const fullPath = resolvePath(path, entry.name); + + if (entry.isDirectory()) { + return [entry.name, read(fullPath)]; + } + const content = fs.readFileSync(fullPath); + const relativePosixPath = relativePath(root, fullPath) + .split(win32.sep) + .join(posix.sep); + + if (shouldReadAsText(relativePosixPath, content)) { + return [entry.name, content.toString('utf8')]; + } + return [entry.name, content]; + }), + ); + } + + return read(root); + } + + clear = (): void => { + this.setContent({}); + }; + + remove = (): void => { + fs.removeSync(this.#root); + }; + + #transformInput(input: MockDirectoryContent[string]): MockEntry[] { + const entries: MockEntry[] = []; + + function traverse(node: MockDirectoryContent[string], path: string) { + const trimmedPath = path.startsWith('/') ? path.slice(1) : path; // trim leading slash + if (typeof node === 'string') { + entries.push({ + type: 'file', + path: trimmedPath, + content: Buffer.from(node, 'utf8'), + }); + } else if (node instanceof Buffer) { + entries.push({ type: 'file', path: trimmedPath, content: node }); + } else { + entries.push({ type: 'dir', path: trimmedPath }); + for (const [name, child] of Object.entries(node)) { + traverse(child, `${trimmedPath}/${name}`); + } + } + } + + traverse(input, ''); + + return entries; + } +} + +/** + * Options for {@link createMockDirectory}. + * + * @public + */ +export interface MockDirectoryOptions { + /** + * In addition to creating a temporary directory, also mock `os.tmpdir()` to return the + * mock directory path until the end of the test suite. + * + * @returns + */ + mockOsTmpDir?: boolean; + + /** + * Initializes the directory with the given content, see {@link MockDirectory.setContent}. + */ + content?: MockDirectoryContent; +} + +/** + * Creates a new temporary mock directory that will be removed after the tests have completed. + * + * @public + * @remarks + * + * This method is intended to be called outside of any test, either at top-level or + * within a `describe` block. It will call `afterAll` to make sure that the mock directory + * is removed after the tests have run. + * + * @example + * ```ts + * describe('MySubject', () => { + * const mockDir = createMockDirectory(); + * + * beforeEach(mockDir.clear); + * + * it('should work', () => { + * // ... use mockDir + * }) + * }) + * ``` + */ +export function createMockDirectory( + options?: MockDirectoryOptions, +): MockDirectory { + const tmpDir = process.env.RUNNER_TEMP || os.tmpdir(); // GitHub Actions + const root = fs.mkdtempSync(joinPath(tmpDir, 'backstage-tmp-test-dir-')); + + const mocker = new MockDirectoryImpl(root); + + const origTmpdir = options?.mockOsTmpDir ? os.tmpdir : undefined; + if (origTmpdir) { + if (Object.hasOwn(origTmpdir, tmpdirMarker)) { + throw new Error( + 'Cannot mock os.tmpdir() when it has already been mocked', + ); + } + const mock = Object.assign(() => mocker.path, { [tmpdirMarker]: true }); + os.tmpdir = mock; + } + + // In CI we expect there to be no need to clean up temporary directories + const needsCleanup = !process.env.CI; + if (needsCleanup) { + process.on('beforeExit', mocker.remove); + } + + try { + afterAll(() => { + if (origTmpdir) { + os.tmpdir = origTmpdir; + } + if (needsCleanup) { + mocker.remove(); + } + }); + } catch { + /* ignore */ + } + + if (options?.content) { + mocker.setContent(options.content); + } + + return mocker; +} diff --git a/packages/backend-test-utils/src/filesystem/index.ts b/packages/backend-test-utils/src/filesystem/index.ts new file mode 100644 index 0000000000..0a4d8c7d00 --- /dev/null +++ b/packages/backend-test-utils/src/filesystem/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + createMockDirectory, + type MockDirectory, + type MockDirectoryOptions, + type MockDirectoryContent, + type MockDirectoryContentOptions, +} from './MockDirectory'; diff --git a/packages/backend-test-utils/src/index.ts b/packages/backend-test-utils/src/index.ts index ae937d2fc8..ff1f2ee460 100644 --- a/packages/backend-test-utils/src/index.ts +++ b/packages/backend-test-utils/src/index.ts @@ -22,5 +22,6 @@ export * from './database'; export * from './msw'; +export * from './filesystem'; export * from './next'; export * from './util'; diff --git a/yarn.lock b/yarn.lock index b7987c9f3a..2c48e783e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3464,7 +3464,6 @@ __metadata: "@types/http-errors": ^2.0.0 "@types/luxon": ^3.0.0 "@types/minimist": ^1.2.0 - "@types/mock-fs": ^4.13.0 "@types/morgan": ^1.9.0 "@types/node-forge": ^1.3.0 "@types/pg": ^8.6.6 @@ -3497,7 +3496,6 @@ __metadata: luxon: ^3.0.0 minimatch: ^5.0.0 minimist: ^1.2.5 - mock-fs: ^5.2.0 morgan: ^1.10.0 msw: ^1.0.0 mysql2: ^2.2.5 @@ -3645,17 +3643,20 @@ __metadata: "@backstage/backend-plugin-api": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" + "@backstage/errors": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/types": "workspace:^" "@types/supertest": ^2.0.8 better-sqlite3: ^8.0.0 express: ^4.17.1 + fs-extra: ^10.0.1 knex: ^2.0.0 msw: ^1.0.0 mysql2: ^2.2.5 pg: ^8.3.0 supertest: ^6.1.3 testcontainers: ^8.1.2 + textextensions: ^5.16.0 uuid: ^8.0.0 peerDependencies: "@types/jest": "*" @@ -40465,10 +40466,10 @@ __metadata: languageName: node linkType: hard -"textextensions@npm:^5.12.0, textextensions@npm:^5.13.0": - version: 5.14.0 - resolution: "textextensions@npm:5.14.0" - checksum: 1f610ccf2a2c1445fb7156c23b5c8defd608c74e8047df18abd4b9ee44c74d29b453ba104d14c53c91f9026670f7c923114cd12200ee5ec458cc518fd0798c74 +"textextensions@npm:^5.12.0, textextensions@npm:^5.13.0, textextensions@npm:^5.16.0": + version: 5.16.0 + resolution: "textextensions@npm:5.16.0" + checksum: d2abd5c962760046aa85d9ca542bd8bdb451370fc0a5e5f807aa80dd2f50175ec10d5ce9d28ae96968aaf6a1b1bea254cf4715f24852d0dcf29c6a60af7f793c languageName: node linkType: hard