Merge pull request #22884 from mttwise/bugfix/fix-aws-s3-dir-structure

Fix AWS S3 Flat Directory Structure on Fetch

.changeset/brown-peaches-hunt.md

@@ -0,0 +1,5 @@
+---
+'@backstage/backend-common': patch
+---
+
+Fix a bug with S3 Fetch that caused all objects to be flattened within a single folder on the local file system.

packages/backend-common/src/reading/AwsS3UrlReader.ts

@@ -46,6 +46,7 @@ import {
 import { AbortController } from '@aws-sdk/abort-controller';
 import { ReadUrlResponseFactory } from './ReadUrlResponseFactory';
 import { Readable } from 'stream';
+import { relative } from 'path/posix';
 
 export const DEFAULT_REGION = 'us-east-1';
 
@@ -345,7 +346,7 @@ export class AwsS3UrlReader implements UrlReader {
 
         responses.push({
           data: s3ObjectData,
-          path: String(allObjects[i]),
+          path: relative(path, String(allObjects[i])),
           lastModifiedAt: response?.LastModified ?? undefined,
         });
       }

packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts

@@ -19,6 +19,7 @@ import path from 'path';
 import { FromReadableArrayOptions } from '../types';
 import { ReadableArrayResponse } from './ReadableArrayResponse';
 import { createMockDirectory } from '@backstage/backend-test-utils';
+import { relative } from 'path/posix';
 
 const name1 = 'file1.yaml';
 const file1 = fs.readFileSync(
@@ -74,9 +75,12 @@ describe('ReadableArrayResponse', () => {
   });
 
   it('should extract entire archive into directory', async () => {
+    const relativePath1 = relative(sourceDir.path, path1);
+    const relativePath2 = relative(sourceDir.path, path2);
+
     const arr: FromReadableArrayOptions = [
-      { data: createReadStream(path1), path: path1 },
-      { data: createReadStream(path2), path: path2 },
+      { data: createReadStream(path1), path: relativePath1 },
+      { data: createReadStream(path2), path: relativePath2 },
     ];
 
     const res = new ReadableArrayResponse(arr, targetDir.path, 'etag');

packages/backend-common/src/reading/tree/ReadableArrayResponse.ts

@@ -15,7 +15,7 @@
  */
 
 import concatStream from 'concat-stream';
-import platformPath, { basename } from 'path';
+import platformPath, { dirname } from 'path';
 
 import getRawBody from 'raw-body';
 import fs from 'fs-extra';
@@ -96,12 +96,9 @@ export class ReadableArrayResponse implements ReadTreeResponse {
 
     for (let i = 0; i < this.stream.length; i++) {
       if (!this.stream[i].path.endsWith('/')) {
-        await pipeline(
-          this.stream[i].data,
-          fs.createWriteStream(
-            platformPath.join(dir, basename(this.stream[i].path)),
-          ),
-        );
+        const filePath = platformPath.join(dir, this.stream[i].path);
+        await fs.mkdir(dirname(filePath), { recursive: true });
+        await pipeline(this.stream[i].data, fs.createWriteStream(filePath));
       }
     }