diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 4fed67cc54..528d84a299 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -67,6 +67,7 @@ "nunjucks": "^3.2.3", "octokit": "^1.7.1", "octokit-plugin-create-pull-request": "^3.10.0", + "p-limit": "^3.1.0", "uuid": "^8.2.0", "winston": "^3.2.1", "yaml": "^1.10.0", diff --git a/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.test.ts b/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.test.ts new file mode 100644 index 0000000000..0e7de7f88d --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.test.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { deserializeDirectoryContents } from './deserializeDirectoryContents'; +import { serializeDirectoryContents } from './serializeDirectoryContents'; + +describe('deserializeDirectoryContents', () => { + beforeEach(() => { + mockFs({ + root: {}, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('deserializes contents into a directory', async () => { + await deserializeDirectoryContents('root', [ + { + path: 'a.txt', + content: Buffer.from('a', 'utf8'), + }, + ]); + await expect(serializeDirectoryContents('root')).resolves.toEqual([ + { + path: 'a.txt', + content: Buffer.from('a', 'utf8'), + executable: false, + }, + ]); + }); + + it('deserializes contents into a deep directory structure', async () => { + await deserializeDirectoryContents('root', [ + { + path: 'a.txt', + content: Buffer.from('a', 'utf8'), + }, + { + path: 'a/b.txt', + content: Buffer.from('b', 'utf8'), + }, + { + path: 'a/b/c.txt', + content: Buffer.from('c', 'utf8'), + }, + ]); + await expect(serializeDirectoryContents('root')).resolves.toEqual([ + { + path: 'a.txt', + content: Buffer.from('a', 'utf8'), + executable: false, + }, + { + path: 'a/b.txt', + content: Buffer.from('b', 'utf8'), + executable: false, + }, + { + path: 'a/b/c.txt', + content: Buffer.from('c', 'utf8'), + executable: false, + }, + ]); + }); +}); diff --git a/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.ts b/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.ts new file mode 100644 index 0000000000..cb567eed9d --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import { dirname } from 'path'; +import { resolveSafeChildPath } from '@backstage/backend-common'; +import { SerializedFile } from './types'; + +/** + * Deserializes a list of serialized files into the target directory. + * + * This method uses `resolveSafeChildPath` to make sure that files are + * not written outside of the target directory. + * + * @internal + */ +export async function deserializeDirectoryContents( + targetPath: string, + files: SerializedFile[], +): Promise { + for (const file of files) { + const filePath = resolveSafeChildPath(targetPath, file.path); + await fs.ensureDir(dirname(filePath)); + await fs.writeFile(filePath, file.content); // Ignore file mode + } +} diff --git a/plugins/scaffolder-backend/src/lib/files/index.ts b/plugins/scaffolder-backend/src/lib/files/index.ts new file mode 100644 index 0000000000..ab6975532f --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { serializeDirectoryContents } from './serializeDirectoryContents'; +export { deserializeDirectoryContents } from './deserializeDirectoryContents'; +export type { SerializedFile } from './types'; diff --git a/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.test.ts b/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.test.ts new file mode 100644 index 0000000000..a58274d694 --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.test.ts @@ -0,0 +1,179 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { serializeDirectoryContents } from './serializeDirectoryContents'; +import mockFs from 'mock-fs'; + +describe('serializeDirectoryContents', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should life files in this directory', async () => { + await expect(serializeDirectoryContents(__dirname)).resolves.toEqual( + expect.arrayContaining([ + { + path: 'index.ts', + executable: false, + content: expect.any(Buffer), + }, + { + path: 'types.ts', + executable: false, + content: expect.any(Buffer), + }, + { + path: 'serializeDirectoryContents.ts', + executable: false, + content: expect.any(Buffer), + }, + { + path: 'serializeDirectoryContents.test.ts', + executable: false, + content: expect.any(Buffer), + }, + ]), + ); + }); + + it('should list files in a mock directory', async () => { + mockFs({ + root: { + 'a.txt': 'a', + b: { + 'b1.txt': 'b1', + 'b2.txt': 'b2', + }, + c: { + c1: { + 'c11.txt': 'c11', + c11: { + 'c111.txt': 'c111', + }, + }, + }, + }, + }); + + await expect(serializeDirectoryContents('root')).resolves.toEqual([ + { + path: 'a.txt', + executable: false, + content: Buffer.from('a', 'utf8'), + }, + { + path: 'b/b1.txt', + executable: false, + content: Buffer.from('b1', 'utf8'), + }, + { + path: 'b/b2.txt', + executable: false, + content: Buffer.from('b2', 'utf8'), + }, + { + path: 'c/c1/c11.txt', + executable: false, + content: Buffer.from('c11', 'utf8'), + }, + { + path: 'c/c1/c11/c111.txt', + executable: false, + content: Buffer.from('c111', 'utf8'), + }, + ]); + }); + + it('should ignore gitignored files', async () => { + mockFs({ + root: { + '.gitignore': '*.txt', + 'a.txt': 'a', + 'a.log': 'a', + }, + }); + + await expect( + serializeDirectoryContents('root', { + gitignore: true, + }), + ).resolves.toEqual([ + { + path: '.gitignore', + executable: false, + content: Buffer.from('*.txt', 'utf8'), + }, + { + path: 'a.log', + executable: false, + content: Buffer.from('a', 'utf8'), + }, + ]); + }); + + it('should use custom glob patterns', async () => { + mockFs({ + root: { + '.a': 'a', + 'a.log': 'a', + 'a.txt': 'a', + b: { + '.b': 'b', + 'b.log': 'b', + 'b.txt': 'b', + }, + c: { + '.c': 'c', + 'c.log': 'c', + 'c.txt': 'c', + }, + }, + }); + + await expect( + serializeDirectoryContents('root', { + gitignore: true, + globPatterns: ['**/*.txt', '*/.?', '*/*.log', '!c/**/.*', '!b/*.log'], + }).then(files => files.sort((a, b) => a.path.localeCompare(b.path))), + ).resolves.toEqual([ + { + path: 'a.txt', + executable: false, + content: Buffer.from('a', 'utf8'), + }, + { + path: 'b/.b', + executable: false, + content: Buffer.from('b', 'utf8'), + }, + { + path: 'b/b.txt', + executable: false, + content: Buffer.from('b', 'utf8'), + }, + { + path: 'c/c.log', + executable: false, + content: Buffer.from('c', 'utf8'), + }, + { + path: 'c/c.txt', + executable: false, + content: Buffer.from('c', 'utf8'), + }, + ]); + }); +}); diff --git a/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.ts b/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.ts new file mode 100644 index 0000000000..608b9e71a5 --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import globby from 'globby'; +import limiterFactory from 'p-limit'; +import { join as joinPath } from 'path'; +import { SerializedFile } from './types'; + +const DEFAULT_GLOB_PATTERNS = ['./**', '!.git']; + +export const isExecutable = (fileMode: number | undefined) => { + if (!fileMode) { + return false; + } + + const executeBitMask = 0o000111; + const res = fileMode & executeBitMask; + return res > 0; +}; + +export async function serializeDirectoryContents( + sourcePath: string, + options?: { + gitignore?: boolean; + globPatterns?: string[]; + }, +): Promise { + const paths = await globby(options?.globPatterns ?? DEFAULT_GLOB_PATTERNS, { + cwd: sourcePath, + dot: true, + gitignore: options?.gitignore, + followSymbolicLinks: false, + objectMode: true, + stats: true, + }); + + const limiter = limiterFactory(10); + + return Promise.all( + paths.map(async ({ path, stats }) => ({ + path, + content: await limiter(async () => + fs.readFile(joinPath(sourcePath, path)), + ), + executable: isExecutable(stats?.mode), + })), + ); +} diff --git a/plugins/scaffolder-backend/src/lib/files/types.ts b/plugins/scaffolder-backend/src/lib/files/types.ts new file mode 100644 index 0000000000..390d6791ee --- /dev/null +++ b/plugins/scaffolder-backend/src/lib/files/types.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface SerializedFile { + path: string; + content: Buffer; + executable?: boolean; +}