paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

scaffolder-backend: add directory serialization utilities

Browse Source 
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
2022-04-04 10:09:05 +02:00
parent 8d53f02bde
commit 91ae2f46d2
7 changed files with 402 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/scaffolder-backend/package.json
+1
View File
@@ -67,6 +67,7 @@
"nunjucks": "^3.2.3",
"octokit": "^1.7.1",
"octokit-plugin-create-pull-request": "^3.10.0",
"p-limit": "^3.1.0",
"uuid": "^8.2.0",
"winston": "^3.2.1",
"yaml": "^1.10.0",
plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.test.ts
+81
View File
@@ -0,0 +1,81 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mockFs from 'mock-fs';
import { deserializeDirectoryContents } from './deserializeDirectoryContents';
import { serializeDirectoryContents } from './serializeDirectoryContents';
describe('deserializeDirectoryContents', () => {
beforeEach(() => {
mockFs({
root: {},
});
});
afterEach(() => {
mockFs.restore();
});
it('deserializes contents into a directory', async () => {
await deserializeDirectoryContents('root', [
{
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
},
]);
await expect(serializeDirectoryContents('root')).resolves.toEqual([
{
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
executable: false,
},
]);
});
it('deserializes contents into a deep directory structure', async () => {
await deserializeDirectoryContents('root', [
{
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
},
{
path: 'a/b.txt',
content: Buffer.from('b', 'utf8'),
},
{
path: 'a/b/c.txt',
content: Buffer.from('c', 'utf8'),
},
]);
await expect(serializeDirectoryContents('root')).resolves.toEqual([
{
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
executable: false,
},
{
path: 'a/b.txt',
content: Buffer.from('b', 'utf8'),
executable: false,
},
{
path: 'a/b/c.txt',
content: Buffer.from('c', 'utf8'),
executable: false,
},
]);
});
});
plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.ts
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import { dirname } from 'path';
import { resolveSafeChildPath } from '@backstage/backend-common';
import { SerializedFile } from './types';
/**
* Deserializes a list of serialized files into the target directory.
*
* This method uses `resolveSafeChildPath` to make sure that files are
* not written outside of the target directory.
*
* @internal
*/
export async function deserializeDirectoryContents(
targetPath: string,
files: SerializedFile[],
): Promise<void> {
for (const file of files) {
const filePath = resolveSafeChildPath(targetPath, file.path);
await fs.ensureDir(dirname(filePath));
await fs.writeFile(filePath, file.content); // Ignore file mode
}
}
plugins/scaffolder-backend/src/lib/files/index.ts
+19
View File
@@ -0,0 +1,19 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { serializeDirectoryContents } from './serializeDirectoryContents';
export { deserializeDirectoryContents } from './deserializeDirectoryContents';
export type { SerializedFile } from './types';
plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.test.ts
+179
View File
@@ -0,0 +1,179 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { serializeDirectoryContents } from './serializeDirectoryContents';
import mockFs from 'mock-fs';
describe('serializeDirectoryContents', () => {
afterEach(() => {
mockFs.restore();
});
it('should life files in this directory', async () => {
await expect(serializeDirectoryContents(__dirname)).resolves.toEqual(
expect.arrayContaining([
{
path: 'index.ts',
executable: false,
content: expect.any(Buffer),
},
{
path: 'types.ts',
executable: false,
content: expect.any(Buffer),
},
{
path: 'serializeDirectoryContents.ts',
executable: false,
content: expect.any(Buffer),
},
{
path: 'serializeDirectoryContents.test.ts',
executable: false,
content: expect.any(Buffer),
},
]),
);
});
it('should list files in a mock directory', async () => {
mockFs({
root: {
'a.txt': 'a',
b: {
'b1.txt': 'b1',
'b2.txt': 'b2',
},
c: {
c1: {
'c11.txt': 'c11',
c11: {
'c111.txt': 'c111',
},
},
},
},
});
await expect(serializeDirectoryContents('root')).resolves.toEqual([
{
path: 'a.txt',
executable: false,
content: Buffer.from('a', 'utf8'),
},
{
path: 'b/b1.txt',
executable: false,
content: Buffer.from('b1', 'utf8'),
},
{
path: 'b/b2.txt',
executable: false,
content: Buffer.from('b2', 'utf8'),
},
{
path: 'c/c1/c11.txt',
executable: false,
content: Buffer.from('c11', 'utf8'),
},
{
path: 'c/c1/c11/c111.txt',
executable: false,
content: Buffer.from('c111', 'utf8'),
},
]);
});
it('should ignore gitignored files', async () => {
mockFs({
root: {
'.gitignore': '*.txt',
'a.txt': 'a',
'a.log': 'a',
},
});
await expect(
serializeDirectoryContents('root', {
gitignore: true,
}),
).resolves.toEqual([
{
path: '.gitignore',
executable: false,
content: Buffer.from('*.txt', 'utf8'),
},
{
path: 'a.log',
executable: false,
content: Buffer.from('a', 'utf8'),
},
]);
});
it('should use custom glob patterns', async () => {
mockFs({
root: {
'.a': 'a',
'a.log': 'a',
'a.txt': 'a',
b: {
'.b': 'b',
'b.log': 'b',
'b.txt': 'b',
},
c: {
'.c': 'c',
'c.log': 'c',
'c.txt': 'c',
},
},
});
await expect(
serializeDirectoryContents('root', {
gitignore: true,
globPatterns: ['**/*.txt', '*/.?', '*/*.log', '!c/**/.*', '!b/*.log'],
}).then(files => files.sort((a, b) => a.path.localeCompare(b.path))),
).resolves.toEqual([
{
path: 'a.txt',
executable: false,
content: Buffer.from('a', 'utf8'),
},
{
path: 'b/.b',
executable: false,
content: Buffer.from('b', 'utf8'),
},
{
path: 'b/b.txt',
executable: false,
content: Buffer.from('b', 'utf8'),
},
{
path: 'c/c.log',
executable: false,
content: Buffer.from('c', 'utf8'),
},
{
path: 'c/c.txt',
executable: false,
content: Buffer.from('c', 'utf8'),
},
]);
});
});
plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.ts
+62
View File
@@ -0,0 +1,62 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import globby from 'globby';
import limiterFactory from 'p-limit';
import { join as joinPath } from 'path';
import { SerializedFile } from './types';
const DEFAULT_GLOB_PATTERNS = ['./**', '!.git'];
export const isExecutable = (fileMode: number | undefined) => {
if (!fileMode) {
return false;
}
const executeBitMask = 0o000111;
const res = fileMode & executeBitMask;
return res > 0;
};
export async function serializeDirectoryContents(
sourcePath: string,
options?: {
gitignore?: boolean;
globPatterns?: string[];
},
): Promise<SerializedFile[]> {
const paths = await globby(options?.globPatterns ?? DEFAULT_GLOB_PATTERNS, {
cwd: sourcePath,
dot: true,
gitignore: options?.gitignore,
followSymbolicLinks: false,
objectMode: true,
stats: true,
});
const limiter = limiterFactory(10);
return Promise.all(
paths.map(async ({ path, stats }) => ({
path,
content: await limiter(async () =>
fs.readFile(joinPath(sourcePath, path)),
),
executable: isExecutable(stats?.mode),
})),
);
}
plugins/scaffolder-backend/src/lib/files/types.ts
+21
View File
@@ -0,0 +1,21 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface SerializedFile {
path: string;
content: Buffer;
executable?: boolean;
}