diff --git a/build/build.py b/build/build.py
index 5214782..2a0f1b0 100644
--- a/build/build.py
+++ b/build/build.py
@@ -141,6 +141,10 @@ def build(
                 if str(file.relative_to(output_dir)) not in built_files:
                     print(f'[PyTeX] Removing old built file {str(file.relative_to(output_dir))}')
                     file.unlink()
+            elif not str(file.relative_to(output_dir)) in built_files:
+                if not file.is_dir() and not str(file.relative_to(output_dir)) == 'build_info.json':
+                    # PyTeX does not at all know something about this file
+                    raise UnknownFileInBuildDirectory(file.relative_to(output_dir))
 
     if write_build_information:
         with open(output_dir / 'build_info.json', 'w') as f:
diff --git a/errors/__init__.py b/errors/__init__.py
index 3377080..3d4a68a 100644
--- a/errors/__init__.py
+++ b/errors/__init__.py
@@ -1,5 +1,6 @@
 from .errors import PyTexError, SubmoduleDirtyForbiddenError, ProgrammingError, ExtraHeaderFileNotFoundError, \
-    UnknownTexVersionError, ModifiedFileInBuildDirectoryError, UnknownFileInBuildDirectoryError
+    UnknownTexVersionError, ModifiedFileInBuildDirectoryError, UnknownFileInBuildDirectoryNoOverwriteError, \
+    UnknownFileInBuildDirectory
 
 __all__ = [
     'PyTexError',
@@ -8,5 +9,6 @@ __all__ = [
     'ExtraHeaderFileNotFoundError',
     'UnknownTexVersionError',
     'ModifiedFileInBuildDirectoryError',
-    'UnknownFileInBuildDirectoryError'
+    'UnknownFileInBuildDirectoryNoOverwriteError',
+    'UnknownFileInBuildDirectory'
 ]
diff --git a/errors/errors.py b/errors/errors.py
index f1cbb9c..e9901c4 100644
--- a/errors/errors.py
+++ b/errors/errors.py
@@ -45,7 +45,7 @@ class ModifiedFileInBuildDirectoryError(PyTexError):
         )
 
 
-class UnknownFileInBuildDirectoryError(PyTexError):
+class UnknownFileInBuildDirectoryNoOverwriteError(PyTexError):
     def __init__(self, filename: str):
         super().__init__(
             f"Unknown file {filename} in build directory found. "
@@ -53,4 +53,13 @@ class UnknownFileInBuildDirectoryError(PyTexError):
             f"Refusing to overwrite this file, since you could lose your data. "
             f"If you are sure, this can be got rid of, delete the file manually, "
             f"and run the build again."
-        )
\ No newline at end of file
+        )
+
+
+class UnknownFileInBuildDirectory(PyTexError):
+    def __init__(self, filename):
+        super().__init__(
+            f"Detected unknown file {filename} in build directory."
+            f"PyTeX has no knowledge about this, you should probably"
+            f"remove it."
+        )
diff --git a/utils/file_integrity.py b/utils/file_integrity.py
index 13fe069..1614549 100644
--- a/utils/file_integrity.py
+++ b/utils/file_integrity.py
@@ -7,7 +7,7 @@ from typing import Optional, List, Dict
 def ensure_file_integrity(file: Path, output_file_name: str, build_info: Optional[List[Dict]] = None):
     if file.exists():
         if not build_info:
-            raise UnknownFileInBuildDirectoryError(str(file))
+            raise UnknownFileInBuildDirectoryNoOverwriteError(str(file))
         found = False
         for info in build_info:
             if info['name'] == output_file_name:
@@ -15,4 +15,4 @@ def ensure_file_integrity(file: Path, output_file_name: str, build_info: Optiona
                     raise ModifiedFileInBuildDirectoryError(str(file))
                 found = True
         if not found:
-            raise UnknownFileInBuildDirectoryError(str(file))
+            raise UnknownFileInBuildDirectoryNoOverwriteError(str(file))