Merge pull request #48 from HEPData/data-file-size-limit

Added file size limit for data files

Author: GraemeWatt

hepdata_validator/full_submission_validator.py

@@ -15,6 +15,8 @@
 from .data_file_validator import DataFileValidator
 
 
+INDIVIDUAL_FILE_SIZE_LIMIT = 10485760
+
 class SchemaType(Enum):
     SUBMISSION = 'submission'
     SINGLE_YAML = 'single file'
@@ -313,6 +315,15 @@ def _check_doc(self, doc):
                 )
                 return is_valid_submission_doc
 
+            file_size = os.path.getsize(data_file_path)
+            if file_size > INDIVIDUAL_FILE_SIZE_LIMIT: # 10MB limit for each file
+                self._add_validation_message(
+                    file=data_file_path,
+                    message=f"Size of data_file '{doc['data_file']}' ({file_size} bytes) is bigger than the limit of " \
+                            f"{INDIVIDUAL_FILE_SIZE_LIMIT} bytes. Try adding the file as an additional_resource instead."
+                )
+                return is_valid_submission_doc
+
             user_data_file_path = self.submission_file_path if self.single_yaml_file else data_file_path
             user_data_file_path = self._remove_temp_directory(user_data_file_path)
 

testsuite/test_data/TestHEPSubmission_invalid/data10.yaml

@@ -30,4 +30,4 @@ dependent_variables:
     errors:
     - {label: stat, symerror: 100}
     - {label: 'sys,detector', symerror: 10}
-    - {label: 'sys,background', symerror: 1}
+    - {label: 'sys,background', symerror: 1}