diff --git a/packages/cli/package.json b/packages/cli/package.json
index 5ac957d5f7..dc2ac1c5f7 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -78,10 +78,10 @@
     "rollup": "2.23.x",
     "rollup-plugin-dts": "1.4.11",
     "rollup-plugin-esbuild": "^2.0.0",
-    "rollup-plugin-image-files": "^1.4.2",
     "rollup-plugin-peer-deps-external": "^2.2.2",
     "rollup-plugin-postcss": "^3.1.1",
     "rollup-plugin-typescript2": "^0.27.3",
+    "rollup-pluginutils": "^2.8.2",
     "start-server-webpack-plugin": "^2.2.5",
     "style-loader": "^1.2.1",
     "sucrase": "^3.14.1",
@@ -114,6 +114,7 @@
     "@types/webpack": "^4.41.7",
     "@types/webpack-dev-server": "^3.10.0",
     "del": "^5.1.0",
+    "mock-fs": "^4.13.0",
     "nodemon": "^2.0.2",
     "ts-node": "^8.6.2"
   },
diff --git a/packages/cli/src/lib/builder/config.ts b/packages/cli/src/lib/builder/config.ts
index afcedaae59..2566460494 100644
--- a/packages/cli/src/lib/builder/config.ts
+++ b/packages/cli/src/lib/builder/config.ts
@@ -22,13 +22,13 @@ import commonjs from '@rollup/plugin-commonjs';
 import resolve from '@rollup/plugin-node-resolve';
 import postcss from 'rollup-plugin-postcss';
 import esbuild from 'rollup-plugin-esbuild';
-import imageFiles from 'rollup-plugin-image-files';
 import svgr from '@svgr/rollup';
 import dts from 'rollup-plugin-dts';
 import json from '@rollup/plugin-json';
 import yaml from '@rollup/plugin-yaml';
 import { RollupOptions, OutputOptions } from 'rollup';
 
+import { forwardFileImports } from './plugins';
 import { BuildOptions, Output } from './types';
 import { paths } from '../paths';
 import { svgrTemplate } from '../svgrTemplate';
@@ -92,7 +92,7 @@ export const makeConfigs = async (
           exclude: [/\/[^/]+\.(?:stories|test)\.[^/]+$/],
         }),
         postcss(),
-        imageFiles({
+        forwardFileImports({
           exclude: /\.icon\.svg$/,
           include: [/\.svg$/, /\.png$/, /\.gif$/, /\.jpg$/, /\.jpeg$/],
         }),
diff --git a/packages/cli/src/lib/builder/plugins.test.ts b/packages/cli/src/lib/builder/plugins.test.ts
new file mode 100644
index 0000000000..41504a700e
--- /dev/null
+++ b/packages/cli/src/lib/builder/plugins.test.ts
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import {
+  NormalizedOutputOptions,
+  OutputAsset,
+  OutputChunk,
+  PluginContext,
+} from 'rollup';
+
+import { forwardFileImports } from './plugins';
+
+const context = {
+  meta: {
+    rollupVersion: '0.0.0',
+    watchMode: false,
+  },
+} as PluginContext;
+
+describe('forwardFileImports', () => {
+  it('should be created', () => {
+    const plugin = forwardFileImports({ include: /\.png$/ });
+    expect(plugin.name).toBe('forward-file-imports');
+  });
+
+  it('should call through to original external option', () => {
+    const plugin = forwardFileImports({ include: /\.png$/ });
+    const external = jest.fn((id: string) => id.endsWith('external'));
+
+    const options = plugin.options?.call(context, { external })!;
+    if (typeof options.external !== 'function') {
+      throw new Error('options.external is not a function');
+    }
+
+    expect(external).toHaveBeenCalledTimes(0);
+    expect(options.external('./my-module', '/dev/src/index.ts', false)).toBe(
+      false,
+    );
+    expect(external).toHaveBeenCalledTimes(1);
+    expect(options.external('./my-external', '/dev/src/index.ts', false)).toBe(
+      true,
+    );
+    expect(external).toHaveBeenCalledTimes(2);
+    expect(options.external('./my-image.png', '/dev/src/index.ts', false)).toBe(
+      true,
+    );
+    expect(external).toHaveBeenCalledTimes(3);
+    expect(options.external('./my-image.png', '/dev/src/index.ts', true)).toBe(
+      true,
+    );
+    expect(external).toHaveBeenCalledTimes(4);
+
+    expect(() =>
+      (options as any).external('./my-image.png', undefined, false),
+    ).toThrow('Unknown importer of file module ./my-image.png');
+  });
+
+  it('should handle original external array', () => {
+    const plugin = forwardFileImports({ include: /\.png$/ });
+
+    const options = plugin.options?.call(context, {
+      external: ['my-external'],
+    })!;
+    if (typeof options.external !== 'function') {
+      throw new Error('options.external is not a function');
+    }
+
+    expect(options.external('my-module', '/dev/src/index.ts', false)).toBe(
+      false,
+    );
+    expect(options.external('my-external', '/dev/src/index.ts', false)).toBe(
+      true,
+    );
+    expect(options.external('my-image.png', '/dev/src/index.ts', false)).toBe(
+      true,
+    );
+  });
+
+  describe('with mock fs', () => {
+    beforeEach(() => {
+      mockFs({
+        '/dev/src/my-module.ts': '',
+        '/dev/src/dir/my-image.png': 'my-image',
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should extract files', async () => {
+      const plugin = forwardFileImports({ include: /\.png$/ });
+
+      const options = plugin.options?.call(context, {})!;
+      if (typeof options.external !== 'function') {
+        throw new Error('options.external is not a function');
+      }
+
+      expect(options.external('./my-module', '/dev/src/index.ts', false)).toBe(
+        false,
+      );
+      expect(
+        options.external('./my-image.png', '/dev/src/dir/index.ts', false),
+      ).toBe(true);
+
+      const outPath = '/dev/dist/dir/my-image.png';
+      await expect(fs.pathExists(outPath)).resolves.toBe(false);
+
+      await plugin.generateBundle?.call(
+        context,
+        { dir: '/dev/dist' } as NormalizedOutputOptions,
+        {
+          ['index.js']: {
+            type: 'chunk',
+            facadeModuleId: '/dev/src/index.ts',
+          } as OutputChunk,
+        },
+        false, // isWrite = false -> no write
+      );
+      await expect(fs.pathExists(outPath)).resolves.toBe(false);
+
+      await plugin.generateBundle?.call(
+        context,
+        { dir: '/dev/dist' } as NormalizedOutputOptions,
+        {
+          // output assets should not cause a write
+          ['index.js']: { type: 'asset' } as OutputAsset,
+          // missing facadeModuleId should not cause a write either
+          ['index2.js']: { type: 'chunk' } as OutputChunk,
+        },
+        true,
+      );
+      await expect(fs.pathExists(outPath)).resolves.toBe(false);
+
+      // output chunk + isWrite -> generate files
+      await plugin.generateBundle?.call(
+        context,
+        { dir: '/dev/dist' } as NormalizedOutputOptions,
+        {
+          ['index.js']: {
+            type: 'chunk',
+            facadeModuleId: '/dev/src/index.ts',
+          } as OutputChunk,
+        },
+        true,
+      );
+      await expect(fs.pathExists(outPath)).resolves.toBe(true);
+
+      // should not break when triggering another write
+      await plugin.generateBundle?.call(
+        context,
+        { file: '/dev/dist/my-output.js' } as NormalizedOutputOptions,
+        {
+          ['index.js']: {
+            type: 'chunk',
+            facadeModuleId: '/dev/src/index.ts',
+          } as OutputChunk,
+        },
+        true,
+      );
+    });
+  });
+});
diff --git a/packages/cli/src/lib/builder/plugins.ts b/packages/cli/src/lib/builder/plugins.ts
new file mode 100644
index 0000000000..48b6b65c61
--- /dev/null
+++ b/packages/cli/src/lib/builder/plugins.ts
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import {
+  dirname,
+  resolve as resolvePath,
+  relative as relativePath,
+} from 'path';
+import { createFilter } from 'rollup-pluginutils';
+import { Plugin, InputOptions, OutputChunk } from 'rollup';
+
+type ForwardFileImportsOptions = {
+  include: Array<string | RegExp> | string | RegExp | null;
+  exclude?: Array<string | RegExp> | string | RegExp | null;
+};
+
+/**
+ * This rollup plugin leaves all encountered asset imports as-is, but
+ * copies the imported files into the output directory.
+ *
+ * For example `import ImageUrl from './my-image.png'` inside `src/MyComponent` will
+ * cause `src/MyComponent/my-image.png` to be copied to the output directory at the
+ * path `dist/MyComponent/my-image.png`. The import itself will stay, but be resolved,
+ * resulting in something like `import ImageUrl from './MyComponent/my-image.png'`
+ */
+export function forwardFileImports(options: ForwardFileImportsOptions): Plugin {
+  const filter = createFilter(options.include, options.exclude);
+
+  // We collect the absolute paths to all files we want to bundle into the
+  // output dir here. Resolving to relative paths in the output dir happens later.
+  const exportedFiles = new Set<string>();
+
+  // We keep track of output directories that we've already copied files
+  // into, so that we don't duplicate that work
+  const generatedFor = new Set<string>();
+
+  return {
+    name: 'forward-file-imports',
+    async generateBundle(outputOptions, bundle, isWrite) {
+      if (!isWrite) {
+        return;
+      }
+
+      const dir = outputOptions.dir || dirname(outputOptions.file!);
+      if (generatedFor.has(dir)) {
+        return;
+      }
+
+      for (const output of Object.values(bundle)) {
+        if (output.type !== 'chunk') {
+          continue;
+        }
+        const chunk = output as OutputChunk;
+
+        // This'll be an absolute path pointing to the initial index file of the
+        // build, and we use it to find the location of the `src` dir
+        if (!chunk.facadeModuleId) {
+          continue;
+        }
+        generatedFor.add(dir);
+
+        // We're assuming that the index file is at the root of the source dir, and
+        // that all assets exist within that dir.
+        const srcRoot = dirname(chunk.facadeModuleId);
+
+        // Copy all the files we found into the dist dir
+        await Promise.all(
+          Array.from(exportedFiles).map(async exportedFile => {
+            const outputPath = relativePath(srcRoot, exportedFile);
+            const targetFile = resolvePath(dir, outputPath);
+
+            await fs.ensureDir(dirname(targetFile));
+            await fs.copyFile(exportedFile, targetFile);
+          }),
+        );
+        return;
+      }
+    },
+    options(inputOptions) {
+      const origExternal = inputOptions.external;
+
+      // We decorate any existing `external` option with our own way of determining
+      // if a module should be external. The can't use `resolveId`, since asset files
+      // aren't passed there, might be some better way to do this though.
+      const external: InputOptions['external'] = (id, importer, isResolved) => {
+        // Call to inner external option
+        if (
+          typeof origExternal === 'function' &&
+          origExternal(id, importer, isResolved)
+        ) {
+          return true;
+        }
+
+        if (Array.isArray(origExternal) && origExternal.includes(id)) {
+          return true;
+        }
+
+        // The piece that we're adding
+        if (!filter(id)) {
+          return false;
+        }
+
+        // Sanity check, dunno if this can happen
+        if (!importer) {
+          throw new Error(`Unknown importer of file module ${id}`);
+        }
+
+        // Resolve relative imports to the full file URL, for deduping and copying later
+        const fullId = isResolved ? id : resolvePath(dirname(importer), id);
+        exportedFiles.add(fullId);
+
+        // Treating this module as external from here, meaning rollup won't try to
+        // put it in the output bundle, but still keep track of the relative imports
+        // as needed in the output code.
+        return true;
+      };
+
+      return { ...inputOptions, external };
+    },
+  };
+}
diff --git a/yarn.lock b/yarn.lock
index badb763132..fe36f17fd2 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10769,7 +10769,7 @@ estraverse@^5.1.0:
   resolved "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz#374309d39fd935ae500e7b92e8a6b4c720e59642"
   integrity sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==
 
-estree-walker@^0.6.0, estree-walker@^0.6.1:
+estree-walker@^0.6.1:
   version "0.6.1"
   resolved "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
   integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
@@ -20234,14 +20234,6 @@ rollup-plugin-esbuild@^2.0.0:
   dependencies:
     "@rollup/pluginutils" "^3.1.0"
 
-rollup-plugin-image-files@^1.4.2:
-  version "1.4.2"
-  resolved "https://registry.npmjs.org/rollup-plugin-image-files/-/rollup-plugin-image-files-1.4.2.tgz#0a329723bace95168a9a6efdacb51370c14fbfe5"
-  integrity sha512-tlydpQdGFssMWhPdWA9SFh4IGVSCzceNgJarJDID+km151IeIVzjATl8ZERNGS/QwMty25iammQqauxl1VUqDQ==
-  dependencies:
-    rollup "0.64.1"
-    rollup-pluginutils "2.4.1"
-
 rollup-plugin-peer-deps-external@^2.2.2:
   version "2.2.3"
   resolved "https://registry.npmjs.org/rollup-plugin-peer-deps-external/-/rollup-plugin-peer-deps-external-2.2.3.tgz#059a8aec1eefb48a475e9fcedc3b9e3deb521213"
@@ -20278,14 +20270,6 @@ rollup-plugin-typescript2@^0.27.3:
     resolve "1.17.0"
     tslib "2.0.1"
 
-rollup-pluginutils@2.4.1:
-  version "2.4.1"
-  resolved "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.4.1.tgz#de43ab54965bbf47843599a7f3adceb723de38db"
-  integrity sha512-wesMQ9/172IJDIW/lYWm0vW0LiKe5Ekjws481R7z9WTRtmO59cqyM/2uUlxvf6yzm/fElFmHUobeQOYz46dZJw==
-  dependencies:
-    estree-walker "^0.6.0"
-    micromatch "^3.1.10"
-
 rollup-pluginutils@^2.8.2:
   version "2.8.2"
   resolved "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e"
@@ -20293,14 +20277,6 @@ rollup-pluginutils@^2.8.2:
   dependencies:
     estree-walker "^0.6.1"
 
-rollup@0.64.1:
-  version "0.64.1"
-  resolved "https://registry.npmjs.org/rollup/-/rollup-0.64.1.tgz#9188ee368e5fcd43ffbc00ec414e72eeb5de87ba"
-  integrity sha512-+ThdVXrvonJdOTzyybMBipP0uz605Z8AnzWVY3rf+cSGnLO7uNkJBlN+9jXqWOomkvumXfm/esmBpA5d53qm7g==
-  dependencies:
-    "@types/estree" "0.0.39"
-    "@types/node" "*"
-
 rollup@2.23.x:
   version "2.23.0"
   resolved "https://registry.npmjs.org/rollup/-/rollup-2.23.0.tgz#b7ab1fee0c0e60132fd0553c4df1e9cdacfada9d"