diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index a887bdc8e7..ee43b74c16 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -125,7 +125,6 @@ "@types/fs-extra": "^9.0.3", "@types/http-errors": "^2.0.0", "@types/minimist": "^1.2.0", - "@types/mock-fs": "^4.13.0", "@types/morgan": "^1.9.0", "@types/node-forge": "^1.3.0", "@types/pg": "^8.6.6", @@ -138,7 +137,6 @@ "aws-sdk-client-mock": "^2.0.0", "better-sqlite3": "^8.0.0", "http-errors": "^2.0.0", - "mock-fs": "^5.2.0", "msw": "^1.0.0", "mysql2": "^2.2.5", "recursive-readdir": "^2.2.2", diff --git a/packages/backend-common/src/reading/AzureUrlReader.test.ts b/packages/backend-common/src/reading/AzureUrlReader.test.ts index 6b2f41a2bf..b6338a1c70 100644 --- a/packages/backend-common/src/reading/AzureUrlReader.test.ts +++ b/packages/backend-common/src/reading/AzureUrlReader.test.ts @@ -23,12 +23,13 @@ import { AzureDevOpsCredentialLike, AzureIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import * as os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { getVoidLogger } from '../logging'; @@ -43,6 +44,8 @@ type AzureIntegrationConfigLike = Partial< const logger = getVoidLogger(); +const mockDir = MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -69,18 +72,8 @@ const urlReaderFactory = (azureIntegration: AzureIntegrationConfigLike) => { ); }; -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('AzureUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(() => mockDir.clear()); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -270,7 +263,7 @@ describe('AzureUrlReader', () => { 'https://dev.azure.com/organization/project/_git/repository', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), diff --git a/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts index c00aeba124..18a186f793 100644 --- a/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketCloudUrlReader.test.ts @@ -19,18 +19,21 @@ import { BitbucketCloudIntegration, readBitbucketCloudIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketCloudUrlReader } from './BitbucketCloudUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; import getRawBody from 'raw-body'; +const mockDir = MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -49,18 +52,8 @@ const reader = new BitbucketCloudUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('BitbucketCloudUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(() => mockDir.clear()); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -316,7 +309,7 @@ describe('BitbucketCloudUrlReader', () => { 'https://bitbucket.org/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -346,7 +339,7 @@ describe('BitbucketCloudUrlReader', () => { 'https://bitbucket.org/backstage/mock/src/master/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts index 9f231e2ba7..82b467f70f 100644 --- a/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketServerUrlReader.test.ts @@ -19,17 +19,20 @@ import { BitbucketServerIntegration, readBitbucketServerIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketServerUrlReader } from './BitbucketServerUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -46,19 +49,7 @@ const reader = new BitbucketServerUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('BitbucketServerUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const worker = setupServer(); setupRequestMockHandlers(worker); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 15e007d516..986619d161 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -19,12 +19,13 @@ import { BitbucketIntegration, readBitbucketIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotModifiedError } from '@backstage/errors'; import { BitbucketUrlReader } from './BitbucketUrlReader'; @@ -68,6 +69,10 @@ describe('BitbucketUrlReader.factory', () => { }); describe('BitbucketUrlReader', () => { + const mockDir = MockDirectory.mockOsTmpDir(); + + beforeEach(() => mockDir.clear()); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -98,18 +103,6 @@ describe('BitbucketUrlReader', () => { { treeResponseFactory }, ); - const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - const worker = setupServer(); setupRequestMockHandlers(worker); @@ -391,7 +384,7 @@ describe('BitbucketUrlReader', () => { 'https://bitbucket.org/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -436,7 +429,7 @@ describe('BitbucketUrlReader', () => { 'https://bitbucket.org/backstage/mock/src/master/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/GerritUrlReader.test.ts b/packages/backend-common/src/reading/GerritUrlReader.test.ts index 19c67ffd56..d3c336ee93 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.test.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.test.ts @@ -14,7 +14,10 @@ * limitations under the License. */ -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import { NotModifiedError, NotFoundError } from '@backstage/errors'; import { @@ -24,7 +27,6 @@ import { import { JsonObject } from '@backstage/types'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import mockFs from 'mock-fs'; import fs from 'fs-extra'; import path from 'path'; import { getVoidLogger } from '../logging'; @@ -33,6 +35,8 @@ import { DefaultReadTreeResponseFactory } from './tree'; import { GerritUrlReader } from './GerritUrlReader'; import getRawBody from 'raw-body'; +const mockDir = MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -82,13 +86,12 @@ const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => { }); }; -// TODO(Rugvip): These tests seem to be a direct or indirect cause of the TaskWorker test flakiness -// We're not sure why at this point, but while investigating these tests are disabled -// eslint-disable-next-line jest/no-disabled-tests -describe.skip('GerritUrlReader', () => { +describe('GerritUrlReader', () => { const worker = setupServer(); setupRequestMockHandlers(worker); + beforeEach(() => mockDir.clear()); + afterAll(() => { jest.clearAllMocks(); }); @@ -249,14 +252,13 @@ describe.skip('GerritUrlReader', () => { path.resolve(__dirname, '__fixtures__/gerrit/gerrit-master-docs.tar.gz'), ); - beforeEach(() => { - mockFs({ - '/tmp/': mockFs.directory(), - '/tmp/gerrit-clone-123abc/repo/mkdocs.yml': mkdocsContent, - '/tmp/gerrit-clone-123abc/repo/docs/first.md': mdContent, + beforeEach(async () => { + await mockDir.setContent({ + 'repo/mkdocs.yml': mkdocsContent, + 'repo/docs/first.md': mdContent, }); const spy = jest.spyOn(fs, 'mkdtemp'); - spy.mockImplementation(() => '/tmp/gerrit-clone-123abc'); + spy.mockImplementation(() => mockDir.path); worker.use( rest.get( @@ -289,7 +291,6 @@ describe.skip('GerritUrlReader', () => { }); afterEach(() => { - mockFs.restore(); jest.clearAllMocks(); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 499f97d6ec..326d3632a4 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -20,12 +20,13 @@ import { GithubIntegration, readGithubIntegrationConfig, } from '@backstage/integration'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { NotFoundError, NotModifiedError } from '@backstage/errors'; import { @@ -37,6 +38,8 @@ import { } from './GithubUrlReader'; import { DefaultReadTreeResponseFactory } from './tree'; +const mockDir = MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -69,21 +72,11 @@ const gheProcessor = new GithubUrlReader( { treeResponseFactory, credentialsProvider: mockCredentialsProvider }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('GithubUrlReader', () => { const worker = setupServer(); setupRequestMockHandlers(worker); - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(() => mockDir.clear()); beforeEach(() => { jest.clearAllMocks(); @@ -420,7 +413,7 @@ describe('GithubUrlReader', () => { 'https://github.com/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -501,7 +494,7 @@ describe('GithubUrlReader', () => { 'https://github.com/backstage/mock/tree/main/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/GitlabUrlReader.test.ts b/packages/backend-common/src/reading/GitlabUrlReader.test.ts index 5af4faf7e5..6500f207f0 100644 --- a/packages/backend-common/src/reading/GitlabUrlReader.test.ts +++ b/packages/backend-common/src/reading/GitlabUrlReader.test.ts @@ -15,12 +15,13 @@ */ import { ConfigReader } from '@backstage/config'; -import { setupRequestMockHandlers } from '@backstage/backend-test-utils'; +import { + MockDirectory, + setupRequestMockHandlers, +} from '@backstage/backend-test-utils'; import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { rest } from 'msw'; import { setupServer } from 'msw/node'; -import os from 'os'; import path from 'path'; import { getVoidLogger } from '../logging'; import { GitlabUrlReader } from './GitlabUrlReader'; @@ -33,6 +34,8 @@ import { const logger = getVoidLogger(); +const mockDir = MockDirectory.mockOsTmpDir(); + const treeResponseFactory = DefaultReadTreeResponseFactory.create({ config: new ConfigReader({}), }); @@ -65,18 +68,8 @@ const hostedGitlabProcessor = new GitlabUrlReader( { treeResponseFactory }, ); -const tmpDir = os.platform() === 'win32' ? 'C:\\tmp' : '/tmp'; - describe('GitlabUrlReader', () => { - beforeEach(() => { - mockFs({ - [tmpDir]: mockFs.directory(), - }); - }); - - afterEach(() => { - mockFs.restore(); - }); + beforeEach(() => mockDir.clear()); const worker = setupServer(); setupRequestMockHandlers(worker); @@ -400,7 +393,7 @@ describe('GitlabUrlReader', () => { 'https://gitlab.com/backstage/mock', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'mkdocs.yml'), 'utf8'), @@ -457,7 +450,7 @@ describe('GitlabUrlReader', () => { 'https://gitlab.com/backstage/mock/tree/main/docs', ); - const dir = await response.dir({ targetDir: tmpDir }); + const dir = await response.dir({ targetDir: mockDir.path }); await expect( fs.readFile(path.join(dir, 'index.md'), 'utf8'), diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index a5728a6169..9f0fd5377e 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -15,33 +15,35 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; -import path, { resolve as resolvePath } from 'path'; +import path from 'path'; import { FromReadableArrayOptions } from '../types'; import { ReadableArrayResponse } from './ReadableArrayResponse'; +import { MockDirectory } from '@backstage/backend-test-utils'; -const path1 = '/file1.yaml'; +const name1 = 'file1.yaml'; const file1 = fs.readFileSync( path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object.yaml'), ); -const path2 = '/file2.yaml'; +const name2 = 'file2.yaml'; const file2 = fs.readFileSync( path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object2.yaml'), ); describe('ReadableArrayResponse', () => { - beforeEach(() => { - mockFs({ - [path1]: file1, - [path2]: file2, - '/tmp': mockFs.directory(), + const sourceDir = MockDirectory.create(); + const targetDir = MockDirectory.create(); + + beforeEach(async () => { + await sourceDir.setContent({ + [name1]: file1, + [name2]: file2, }); + await targetDir.clear(); }); - afterEach(() => { - mockFs.restore(); - }); + const path1 = sourceDir.resolve(name1); + const path2 = sourceDir.resolve(name2); it('should read files', async () => { const arr: FromReadableArrayOptions = [ @@ -49,7 +51,7 @@ describe('ReadableArrayResponse', () => { { data: fs.createReadStream(path2), path: path2 }, ]; - const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); + const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -57,10 +59,7 @@ describe('ReadableArrayResponse', () => { { path: path2, content: expect.any(Function) }, ]); const contents = await Promise.all(files.map(f => f.content())); - expect(contents.map(c => c.toString('utf8').trim())).toEqual([ - 'site_name: Test', - 'site_name: Test2', - ]); + expect(contents).toEqual([file1, file2]); }); it('should extract entire archive into directory', async () => { @@ -69,14 +68,12 @@ describe('ReadableArrayResponse', () => { { data: fs.createReadStream(path2), path: path2 }, ]; - const res = new ReadableArrayResponse(arr, '/tmp', 'etag'); + const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); const dir = await res.dir(); - await expect( - fs.readFile(resolvePath(dir, 'file1.yaml'), 'utf8'), - ).resolves.toMatch(/site_name: Test/); - await expect( - fs.readFile(resolvePath(dir, 'file2.yaml'), 'utf8'), - ).resolves.toMatch(/site_name: Test2/); + await expect(targetDir.content({ path: dir })).resolves.toEqual({ + [name1]: file1.toString('utf8'), + [name2]: file2.toString('utf8'), + }); }); }); diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts index 1765a41156..ce126eb345 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.test.ts @@ -15,30 +15,31 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { resolve as resolvePath } from 'path'; import { TarArchiveResponse } from './TarArchiveResponse'; +import { MockDirectory } from '@backstage/backend-test-utils'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.tar.gz'), ); describe('TarArchiveResponse', () => { - beforeEach(() => { - mockFs({ - '/test-archive.tar.gz': archiveData, - '/tmp': mockFs.directory(), - }); - }); + const sourceDir = MockDirectory.create(); + const targetDir = MockDirectory.create(); - afterEach(() => { - mockFs.restore(); + beforeAll(async () => { + await sourceDir.setContent({ 'test-archive.tar.gz': archiveData }); + }); + beforeEach(async () => { + await targetDir.clear(); }); it('should read files', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -61,10 +62,16 @@ describe('TarArchiveResponse', () => { }); it('should read files with filter', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -80,16 +87,18 @@ describe('TarArchiveResponse', () => { }); it('should read as archive and files', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new TarArchiveResponse(buffer, '', '/tmp', 'etag'); + const res2 = new TarArchiveResponse(buffer, '', targetDir.path, 'etag'); const files = await res2.files(); expect(files).toEqual([ @@ -112,9 +121,11 @@ describe('TarArchiveResponse', () => { }); it('should extract entire archive into directory', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, '', targetDir.path, 'etag'); const dir = await res.dir(); await expect( fs.readFile(resolvePath(dir, 'mkdocs.yml'), 'utf8'), @@ -125,67 +136,91 @@ describe('TarArchiveResponse', () => { }); it('should extract archive into directory with a subpath', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, 'docs', '/tmp', 'etag'); + const res = new TarArchiveResponse(stream, 'docs', targetDir.path, 'etag'); const dir = await res.dir(); - expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); - await expect( - fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), - ).resolves.toBe('# Test\n'); + await expect(targetDir.content({ path: dir })).resolves.toEqual({ + 'index.md': '# Test\n', + }); }); it('should extract archive into directory with a subpath and filter', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); - - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), ); - const dir = await res.dir({ targetDir: '/tmp' }); - expect(dir).toBe('/tmp'); - await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( - true, + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); - await expect( - fs.pathExists(resolvePath(dir, 'docs/index.md')), - ).resolves.toBe(false); + + await targetDir.addContent({ sub: {} }); + const dir = await res.dir({ targetDir: targetDir.resolve('sub') }); + + expect(dir).toBe(targetDir.resolve('sub')); + await expect(targetDir.content()).resolves.toEqual({ + sub: { + 'mkdocs.yml': 'site_name: Test\n', + }, + }); }); - it('should leave temporary directories in place in the case of an error', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + it('should clean up temporary directories in place in the case of an error', async () => { + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', () => { - throw new Error('NOPE'); - }); + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + () => { + throw new Error('NOPE'); + }, + ); - const tmpDir = await fs.mkdtemp('/tmp/test'); - // selects the wrong overload by default - const mkdtemp = jest.spyOn(fs, 'mkdtemp') as unknown as jest.SpyInstance< - Promise, - [] - >; - mkdtemp.mockResolvedValue(tmpDir); + await targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); + const mkdtemp = jest + .spyOn(fs, 'mkdtemp') + .mockImplementation(async () => sub); + + await expect(fs.pathExists(sub)).resolves.toBe(true); await expect(res.dir()).rejects.toThrow('NOPE'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(false); + await expect(fs.pathExists(sub)).resolves.toBe(false); mkdtemp.mockRestore(); }); it('should leave directory in place if provided in the case of an error', async () => { - const stream = fs.createReadStream('/test-archive.tar.gz'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive.tar.gz'), + ); - const res = new TarArchiveResponse(stream, '', '/tmp', 'etag', () => { - throw new Error('NOPE'); - }); + const res = new TarArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + () => { + throw new Error('NOPE'); + }, + ); - const tmpDir = await fs.mkdtemp('/tmp/test'); + await targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); - await expect(res.dir({ targetDir: tmpDir })).rejects.toThrow('NOPE'); - await expect(fs.pathExists(tmpDir)).resolves.toBe(true); + await expect(fs.pathExists(sub)).resolves.toBe(true); + await expect(res.dir({ targetDir: sub })).rejects.toThrow('NOPE'); + await expect(fs.pathExists(sub)).resolves.toBe(true); }); }); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts index 0b010565dd..20cf33241b 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts @@ -15,11 +15,11 @@ */ import fs from 'fs-extra'; -import mockFs from 'mock-fs'; import { Readable } from 'stream'; import { create as createArchive } from 'archiver'; import { resolve as resolvePath } from 'path'; import { ZipArchiveResponse } from './ZipArchiveResponse'; +import { MockDirectory } from '@backstage/backend-test-utils'; const archiveData = fs.readFileSync( resolvePath(__filename, '../../__fixtures__/mock-main.zip'), @@ -35,24 +35,25 @@ const archiveWithMaliciousEntry = fs.readFileSync( ); describe('ZipArchiveResponse', () => { - beforeEach(() => { - mockFs({ - '/test-archive.zip': archiveData, - '/test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, - '/test-archive-corrupted.zip': archiveDataCorrupted, - '/test-archive-malicious.zip': archiveWithMaliciousEntry, - '/tmp': mockFs.directory(), + const sourceDir = MockDirectory.create(); + const targetDir = MockDirectory.create(); + + beforeAll(async () => { + await sourceDir.setContent({ + 'test-archive.zip': archiveData, + 'test-archive-with-extra-root-dir.zip': archiveDataWithExtraDir, + 'test-archive-corrupted.zip': archiveDataCorrupted, + 'test-archive-malicious.zip': archiveWithMaliciousEntry, }); }); - - afterEach(() => { - mockFs.restore(); + beforeEach(async () => { + await targetDir.clear(); }); it('should read files', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const files = await res.files(); expect(files).toEqual([ @@ -76,10 +77,14 @@ describe('ZipArchiveResponse', () => { }); it('should read files with filter', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); const files = await res.files(); @@ -95,16 +100,16 @@ describe('ZipArchiveResponse', () => { }); it('should read as archive and files', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const buffer = await res.archive(); await expect(res.archive()).rejects.toThrow( 'Response has already been read', ); - const res2 = new ZipArchiveResponse(buffer, '', '/tmp', 'etag'); + const res2 = new ZipArchiveResponse(buffer, '', targetDir.path, 'etag'); const files = await res2.files(); expect(files).toEqual([ @@ -127,9 +132,9 @@ describe('ZipArchiveResponse', () => { }); it('should extract entire archive into directory', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const dir = await res.dir(); await expect( @@ -141,38 +146,45 @@ describe('ZipArchiveResponse', () => { }); it('should extract archive into directory with a subpath', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); + + const res = new ZipArchiveResponse(stream, 'docs/', targetDir.path, 'etag'); - const res = new ZipArchiveResponse(stream, 'docs/', '/tmp', 'etag'); const dir = await res.dir(); - expect(dir).toMatch(/^[\/\\]tmp[\/\\].*$/); - await expect( - fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), - ).resolves.toBe('# Test\n'); + await expect(targetDir.content({ path: dir })).resolves.toEqual({ + 'index.md': '# Test\n', + }); }); it('should extract archive into directory with a subpath and filter', async () => { - const stream = fs.createReadStream('/test-archive.zip'); + const stream = fs.createReadStream(sourceDir.resolve('test-archive.zip')); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag', path => - path.endsWith('.yml'), + const res = new ZipArchiveResponse( + stream, + '', + targetDir.path, + 'etag', + path => path.endsWith('.yml'), ); - const dir = await res.dir({ targetDir: '/tmp' }); - expect(dir).toBe('/tmp'); - await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe( - true, - ); - await expect( - fs.pathExists(resolvePath(dir, 'docs/index.md')), - ).resolves.toBe(false); + await targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); + const dir = await res.dir({ targetDir: sub }); + + expect(dir).toBe(sub); + + await expect(targetDir.content()).resolves.toEqual({ + sub: { + 'mkdocs.yml': 'site_name: Test\n', + }, + }); }); it('should extract a large archive', async () => { const fileCount = 10; const fileSize = 1000 * 1000; const filePath = await new Promise((resolve, reject) => { - const outFile = '/large-archive.zip'; + const outFile = targetDir.resolve('large-archive.zip'); const archive = createArchive('zip'); archive.on('error', reject); @@ -195,12 +207,14 @@ describe('ZipArchiveResponse', () => { const stream = fs.createReadStream(filePath); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); + + await targetDir.addContent({ sub: {} }); + const sub = targetDir.resolve('sub'); const dir = await res.dir({ - targetDir: '/out', + targetDir: sub, }); - expect(dir).toBe('/out'); const files = await fs.readdir(dir); expect(files).toHaveLength(fileCount); @@ -211,9 +225,11 @@ describe('ZipArchiveResponse', () => { }); it('should throw on invalid archive', async () => { - const stream = fs.createReadStream('/test-archive-corrupted.zip'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive-corrupted.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); const filesPromise = res.files(); await expect(filesPromise).rejects.toThrow( @@ -222,18 +238,22 @@ describe('ZipArchiveResponse', () => { }); it('should throw on entries with a path outside the destination dir', async () => { - const stream = fs.createReadStream('/test-archive-malicious.zip'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive-malicious.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); await expect(res.files()).rejects.toThrow( 'invalid relative path: ../side.txt', ); }); it('should throw on entries that attempt to write outside destination dir', async () => { - const stream = fs.createReadStream('/test-archive-malicious.zip'); + const stream = fs.createReadStream( + sourceDir.resolve('test-archive-malicious.zip'), + ); - const res = new ZipArchiveResponse(stream, '', '/tmp', 'etag'); + const res = new ZipArchiveResponse(stream, '', targetDir.path, 'etag'); await expect(res.dir()).rejects.toThrow( 'invalid relative path: ../side.txt', ); diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 1e55150310..cd57788743 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -94,7 +94,10 @@ export class ZipArchiveResponse implements ReadTreeResponse { return new Promise((resolve, reject) => { writeStream.on('error', reject); writeStream.on('finish', () => - resolve({ fileName: tmpFile, cleanup: () => fs.remove(tmpFile) }), + resolve({ + fileName: tmpFile, + cleanup: () => fs.rm(tmpDir, { recursive: true }), + }), ); stream.pipe(writeStream); }); @@ -152,7 +155,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { }); }); - temporary.cleanup(); + await temporary.cleanup(); return files; } @@ -175,7 +178,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { archive.finalize(); - temporary.cleanup(); + await temporary.cleanup(); return archive; } @@ -204,7 +207,7 @@ export class ZipArchiveResponse implements ReadTreeResponse { }); }); - temporary.cleanup(); + await temporary.cleanup(); return dir; } diff --git a/packages/backend-common/src/util/DockerContainerRunner.test.ts b/packages/backend-common/src/util/DockerContainerRunner.test.ts index 3208679d1f..6b4150540a 100644 --- a/packages/backend-common/src/util/DockerContainerRunner.test.ts +++ b/packages/backend-common/src/util/DockerContainerRunner.test.ts @@ -14,27 +14,24 @@ * limitations under the License. */ +import fs from 'fs-extra'; import Docker from 'dockerode'; -import mockFs from 'mock-fs'; -import os from 'os'; -import path from 'path'; import Stream, { PassThrough } from 'stream'; import { ContainerRunner } from './ContainerRunner'; import { DockerContainerRunner, UserOptions } from './DockerContainerRunner'; +import { MockDirectory } from '@backstage/backend-test-utils'; const mockDocker = new Docker() as jest.Mocked; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; describe('DockerContainerRunner', () => { let containerTaskApi: ContainerRunner; - beforeEach(() => { - mockFs({ - [rootDir]: { - input: mockFs.directory(), - output: mockFs.directory(), - }, - }); + const inputDir = MockDirectory.create(); + const outputDir = MockDirectory.create(); + + beforeEach(async () => { + await inputDir.clear(); + await outputDir.clear(); jest.spyOn(mockDocker, 'pull').mockImplementation((async ( _image: string, @@ -59,16 +56,15 @@ describe('DockerContainerRunner', () => { afterEach(() => { jest.clearAllMocks(); - mockFs.restore(); }); const imageName = 'dockerOrg/image'; const args = ['bash', '-c', 'echo test']; const mountDirs = { - [path.join(rootDir, 'input')]: '/input', - [path.join(rootDir, 'output')]: '/output', + [inputDir.path]: '/input', + [outputDir.path]: '/output', }; - const workingDir = path.join(rootDir, 'input'); + const workingDir = inputDir.path; const envVars = { HOME: '/tmp', LOG_LEVEL: 'debug' }; const envVarsArray = ['HOME=/tmp', 'LOG_LEVEL=debug']; @@ -117,8 +113,8 @@ describe('DockerContainerRunner', () => { HostConfig: { AutoRemove: true, Binds: expect.arrayContaining([ - `${path.join(rootDir, 'input')}:/input`, - `${path.join(rootDir, 'output')}:/output`, + `${await fs.realpath(inputDir.path)}:/input`, + `${await fs.realpath(outputDir.path)}:/output`, ]), }, Volumes: { diff --git a/yarn.lock b/yarn.lock index 86a011100e..dc8e58da96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3464,7 +3464,6 @@ __metadata: "@types/http-errors": ^2.0.0 "@types/luxon": ^3.0.0 "@types/minimist": ^1.2.0 - "@types/mock-fs": ^4.13.0 "@types/morgan": ^1.9.0 "@types/node-forge": ^1.3.0 "@types/pg": ^8.6.6 @@ -3497,7 +3496,6 @@ __metadata: luxon: ^3.0.0 minimatch: ^5.0.0 minimist: ^1.2.5 - mock-fs: ^5.2.0 morgan: ^1.10.0 msw: ^1.0.0 mysql2: ^2.2.5