Do more robust handling of download failures.

Author: rtibbles

ricecooker/utils/pipeline/file_handler.py

@@ -252,6 +252,12 @@ def execute(
 
             file_metadata.path = path
 
+            # Ensure the file was actually processed and is in storage
+            if not path.startswith(os.path.abspath(config.STORAGE_DIRECTORY)):
+                raise InvalidFileException(
+                    f"File handler failed to process {original_path} - result not in storage"
+                )
+
             self._output_path = None
 
             file_metadata_list.append(file_metadata)

ricecooker/utils/pipeline/transfer.py

@@ -434,7 +434,4 @@ def execute(
             # The download stage is special, as we expect it to always return a file
             # if it does not, we raise an exception to prevent further processing
             raise InvalidFileException(f"No file could be downloaded from {path}")
-        for metadata in metadata_list:
-            if metadata.path == path:
-                raise InvalidFileException(f"{path} failed to transfer")
         return metadata_list