Merge pull request #44 from riga/fix/extended_attribute_files

Add check for extended attribute files in validator.

Author: GraemeWatt

hepdata_validator/full_submission_validator.py

@@ -205,27 +205,48 @@ def validate(self, directory=None, file=None, archive=None):
 
             # Check all files in directory are in included_files
             if not self.single_yaml_file and self.schema_version >= packaging_version.parse("1.1.0"):
+                # helper to check if a provided file is not meant to describe HEP data, but rather
+                # represents "extended attributes" (e.g.) as a result of BSD tar (default on MacOS)
+                # which creates these extra files when archiving files with extended attributes on
+                # HSF+ volumes (denoted by "@" in permission bits)
+                def is_ext_attr_file(f):
+                    # three conditions must be fulfilled
+                    # 1. the file must not be referenced in the submission (already checked below)
+                    # 2. the file name must have the format "._<actual_file>"
+                    prefix = "._"
+                    if not f.startswith(prefix):
+                        return False
+                    # 3. a file named "<actual_file>" must exist in the same directory
+                    if not os.path.isfile(os.path.join(self.directory, f[len(prefix):])):
+                        return False
+                    return True
+
                 for f in os.listdir(self.directory):
                     file_path = os.path.join(self.directory, f)
                     if file_path not in self.included_files:
                         self._add_validation_message(
-                            file=file_path, message='%s is not referenced in the submission.' % f
+                            file=file_path, message=f'{f} is not referenced in the submission.'
                         )
+                        if is_ext_attr_file(f):
+                            self._add_validation_message(
+                               file=file_path, message=f'{f} might be a file created by tar on MacOS. Set COPYFILE_DISABLE=1 before creating the archive.',
+                               level='hint'
+                            )
 
             return len(self.messages) == 0
         finally:
             if self.temp_directory:
                 # Delete temporary Directory
                 shutil.rmtree(self.temp_directory)
 
-    def _add_validation_message(self, file, message):
+    def _add_validation_message(self, file, message, **kwargs):
         if self.temp_directory:
             # Remove temp directory from filename and message
             file = self._remove_temp_directory(file)
             message = self._remove_temp_directory(message)
 
         self.add_validation_message(ValidationMessage(
-            file=file, message=message
+            file=file, message=message, **kwargs
         ))
 
     def _remove_temp_directory(self, s):

testsuite/test_data/TestHEPSubmission_invalid.zip

@@ -0,0 +1,33 @@
+independent_variables:
+- header: {name: M(ZZ), units: GEV}
+  values:
+  - {low: 0, high: 240}
+  - {low: 240, high: 300}
+  - {low: 300, high: 400}
+  - {low: 400, high: 800}
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DM(ZZ)}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GeV, value: 7000}
+  values:
+  - value: 2200
+    errors:
+    - {label: stat, symerror: 300}
+    - {label: 'sys,detector', symerror: 40}
+    - {label: 'sys,background', symerror: 2}
+  - value: 4500
+    errors:
+    - {label: stat, symerror: 1000}
+    - {label: 'sys,detector', symerror: 100}
+    - {label: 'sys,background', symerror: 5}
+  - value: 1000
+    errors:
+    - {label: stat, symerror: 400}
+    - {label: 'sys,detector', symerror: 20}
+    - {label: 'sys,background', symerror: 2}
+  - value: 280
+    errors:
+    - {label: stat, symerror: 100}
+    - {label: 'sys,detector', symerror: 10}
+    - {label: 'sys,background', symerror: 1}

testsuite/test_data/TestHEPSubmission_invalid/._data10.yaml

@@ -0,0 +1,33 @@
+independent_variables:
+- header: {name: M(ZZ), units: GEV}
+  values:
+  - {low: 0, high: 240}
+  - {low: 240, high: 300}
+  - {low: 300, high: 400}
+  - {low: 400, high: 800}
+dependent_variables:
+- header: {name: 10**6 * 1/SIG(fiducial) * D(SIG(fiducial))/DM(ZZ)}
+  qualifiers:
+  - {name: RE, value: P P --> Z0 < LEPTON+ LEPTON- > Z0 < LEPTON+ LEPTON- > X}
+  - {name: SQRT(S), units: GeV, value: 7000}
+  values:
+  - value: 2200
+    errors:
+    - {label: stat, symerror: 300}
+    - {label: 'sys,detector', symerror: 40}
+    - {label: 'sys,background', symerror: 2}
+  - value: 4500
+    errors:
+    - {label: stat, symerror: 1000}
+    - {label: 'sys,detector', symerror: 100}
+    - {label: 'sys,background', symerror: 5}
+  - value: 1000
+    errors:
+    - {label: stat, symerror: 400}
+    - {label: 'sys,detector', symerror: 20}
+    - {label: 'sys,background', symerror: 2}
+  - value: 280
+    errors:
+    - {label: stat, symerror: 100}
+    - {label: 'sys,detector', symerror: 10}
+    - {label: 'sys,background', symerror: 1}

testsuite/test_data/TestHEPSubmission_invalid/._data11.yaml

@@ -152,4 +152,18 @@ data_file: data8.yaml
 additional_resources:
 - {description: Image file, location: figFigure10B.png}
 - {description: Thumbnail image file, location: thumb_figFigure10B.png}
+---
+# This is Table 9.
+name: "Table 9"
+location: Data from Figure 10B
+description: Table with a weird but accepted data_file name
+keywords: []
+data_file: ._data9.yaml
+---
+# This is Table 10.
+name: "Table 10"
+location: Data from Figure 10B
+description: Table whose data_file is accompanied by a file representing extended arguments which should lead to a hint.
+keywords: []
+data_file: data10.yaml
 # End of YAML file.

testsuite/test_data/TestHEPSubmission_invalid/._data9.yaml

@@ -247,7 +247,7 @@ def test_invalid_data_directory(validator_v1, data_path, capsys):
     is_valid = validator_v1.validate(directory=dir)
     assert not is_valid
     expected_valid_files = [os.path.join(dir, f) for f in [
-        'data1.yaml', 'data4.yaml', 'data5.yaml', 'data6.yaml', 'data7.yaml'
+        'data1.yaml', 'data4.yaml', 'data5.yaml', 'data6.yaml', 'data7.yaml', '._data9.yaml', 'data10.yaml'
     ]]
     assert validator_v1.valid_files == {SchemaType.DATA: expected_valid_files}
     assert validator_v1.has_errors
@@ -257,7 +257,9 @@ def test_invalid_data_directory(validator_v1, data_path, capsys):
         os.path.join(dir, 'submission.yaml'),
         os.path.join(dir, 'data3.yaml'),
         os.path.join(dir, 'data8.yaml'),
-        os.path.join(dir, 'figFigure8B.png')
+        os.path.join(dir, 'figFigure8B.png'),
+        os.path.join(dir, '._data10.yaml'),
+        os.path.join(dir, '._data11.yaml')
     ]
     assert set(errors.keys()) == set(expected_file_names)
     assert errors[expected_file_names[0]][0].message == "Name of data_file 'mydirectory/data2.yaml' should not contain '/'."
@@ -270,6 +272,11 @@ def test_invalid_data_directory(validator_v1, data_path, capsys):
 		did not find expected key
 		  in "{dir}/data8.yaml", line 9, column 3"""
     assert errors[expected_file_names[3]][0].message == f"figFigure8B.png is not referenced in the submission."
+    assert len(errors[expected_file_names[4]]) == 2
+    assert errors[expected_file_names[4]][0].message == f"._data10.yaml is not referenced in the submission."
+    assert errors[expected_file_names[4]][1].message == f"._data10.yaml might be a file created by tar on MacOS. Set COPYFILE_DISABLE=1 before creating the archive."
+    assert errors[expected_file_names[4]][1].level == 'hint'
+    assert errors[expected_file_names[5]][0].message == f"._data11.yaml is not referenced in the submission."
 
 
 def test_invalid_archive(validator_v1, data_path):#, capsys):
@@ -278,7 +285,7 @@ def test_invalid_archive(validator_v1, data_path):#, capsys):
     is_valid = validator_v1.validate(archive=archive)
     assert not is_valid
     expected_valid_files = [os.path.join(dir, f) for f in [
-        'data1.yaml', 'data4.yaml', 'data5.yaml', 'data6.yaml', 'data7.yaml'
+        'data1.yaml', 'data4.yaml', 'data5.yaml', 'data6.yaml', 'data7.yaml', '._data9.yaml', 'data10.yaml'
     ]]
     assert validator_v1.valid_files == {SchemaType.DATA: expected_valid_files}
     assert validator_v1.has_errors
@@ -288,7 +295,9 @@ def test_invalid_archive(validator_v1, data_path):#, capsys):
         os.path.join(dir, 'submission.yaml'),
         os.path.join(dir, 'data3.yaml'),
         os.path.join(dir, 'data8.yaml'),
-        os.path.join(dir, 'figFigure8B.png')
+        os.path.join(dir, 'figFigure8B.png'),
+        os.path.join(dir, '._data10.yaml'),
+        os.path.join(dir, '._data11.yaml')
     ]
     assert set(errors.keys()) == set(expected_file_names)
     assert errors[expected_file_names[0]][0].message == "Name of data_file 'mydirectory/data2.yaml' should not contain '/'."
@@ -301,6 +310,11 @@ def test_invalid_archive(validator_v1, data_path):#, capsys):
 		did not find expected key
 		  in "{dir}/data8.yaml", line 9, column 3"""
     assert errors[expected_file_names[3]][0].message == f"figFigure8B.png is not referenced in the submission."
+    assert len(errors[expected_file_names[4]]) == 2
+    assert errors[expected_file_names[4]][0].message == f"._data10.yaml is not referenced in the submission."
+    assert errors[expected_file_names[4]][1].message == f"._data10.yaml might be a file created by tar on MacOS. Set COPYFILE_DISABLE=1 before creating the archive."
+    assert errors[expected_file_names[4]][1].level == 'hint'
+    assert errors[expected_file_names[5]][0].message == f"._data11.yaml is not referenced in the submission."
 
 
 def test_invalid_syntax_submission(validator_v1, data_path, capsys):