From ad76f3e6103b11301159660561819c0fbb0c7b05 Mon Sep 17 00:00:00 2001 From: Simeon Warner Date: Wed, 24 Mar 2021 16:18:34 -0400 Subject: [PATCH] Release 1.2.0 (#32) * Add deep test object * Add storage root tests for E072, E072, E003d, E004a, E004b (#27) * Add standalone inventory validation (#31) --- CHANGES.md | 4 ++++ README | 10 +++++----- ocfl-validate.py | 21 +++++++++++++++------ ocfl/_version.py | 2 +- ocfl/object.py | 25 ++++++++++++++++++++++++- ocfl/object_utils.py | 35 +++++++++++++++++++++++++++-------- tests/test_namaste.py | 8 ++++++++ tests/test_object.py | 27 ++++++++++++++++++++++----- tests/test_object_utils.py | 25 +++++++++++++++++++++---- 9 files changed, 127 insertions(+), 30 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index b9fde809..e1e48be7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # ocfl-py changelog +## 2021-03-24 v1.2.0 + + * Add ability for `ocfl-validate.py` to validate a standalone inventory + ## 2020-09-22 v1.1.1 * Add deeply nested text object (`extra_fixtures/good-objects/ten_level_deep_directories.zip`) diff --git a/README b/README index c23d123c..5f2064ef 100644 --- a/README +++ b/README @@ -2,8 +2,8 @@ ocfl-py - An OCFL implementation in Python ========================================== -.. image:: https://travis-ci.org/zimeon/ocfl-py.svg?branch=main - :target: https://travis-ci.org/zimeon/ocfl-py +.. image:: https://travis-ci.com/zimeon/ocfl-py.svg?branch=main + :target: https://travis-ci.com/zimeon/ocfl-py .. image:: https://coveralls.io/repos/github/zimeon/ocfl-py/badge.svg?branch=main :target: https://coveralls.io/github/zimeon/ocfl-py?branch=main @@ -19,7 +19,7 @@ Installing This code requires Python 3. -This code attempts to support the OCFL specification v1.0 and to additional +This code attempts to support the OCFL specification v1.0 and additional developments. To get the most up to date version check out the ``main`` branch from github (or if you are reckless you can try the ``develop`` branch). @@ -34,7 +34,7 @@ Use There should then be three scripts available: -- ``ocfl-validate.py`` - validate OCFL objects or OCFL storage roots +- ``ocfl-validate.py`` - validate OCFL objects, OCFL storage roots or standalone OCFL inventory files - ``ocfl-object.py`` - build, manipulate, extract from or validate an OCFL object - ``ocfl-store.py`` - add or access OCFL objects under an OCFL storage root @@ -59,7 +59,7 @@ for guidelines for contributing. Copyright and License --------------------- -Copyright 2018--2020 Simeon Warner and `contributors +Copyright 2018--2021 Simeon Warner and `contributors `_. Provided under the MIT license, see `LICENSE.txt `_. diff --git a/ocfl-validate.py b/ocfl-validate.py index 8d2aee9b..0edc40cf 100755 --- a/ocfl-validate.py +++ b/ocfl-validate.py @@ -6,11 +6,12 @@ import sys parser = argparse.ArgumentParser( - description='Validate one or more OCFL objects or storage roots. By default ' - 'shows any errors or warnings, and final validation status. Use -q to show ' - 'only errors, -Q to show only validation status.') + description='Validate one or more OCFL objects, storage roots or standalone ' + 'inventory files. By default shows any errors or warnings, and final ' + 'validation status. Use -q to show only errors, -Q to show only validation ' + 'status.') parser.add_argument('path', type=str, nargs='*', - help='OCFL object or storage root path(s) to validate') + help='OCFL object, storage root or inventory path(s) to validate') parser.add_argument('--quiet', '-q', action='store_true', help="Be quiet, do not show warnings") parser.add_argument('--very-quiet', '-Q', action='store_true', @@ -34,6 +35,7 @@ num = 0 num_good = 0 num_paths = len(args.path) +show_warnings = not args.quiet and not args.very_quiet for path in args.path: num += 1 path_type = ocfl.find_path_type(path) @@ -41,7 +43,7 @@ log.info("Validating OCFL Object at " + path) obj = ocfl.Object(lax_digests=args.lax_digests) if obj.validate(path, - show_warnings=not args.quiet and not args.very_quiet, + show_warnings=show_warnings, show_errors=not args.very_quiet, check_digests=not args.no_check_digests): num_good += 1 @@ -49,10 +51,17 @@ log.info("Validating OCFL Storage Root at " + path) store = ocfl.Store(root=path, lax_digests=args.lax_digests) - if store.validate(show_warnings=not args.quiet and not args.very_quiet, + if store.validate(show_warnings=show_warnings, show_errors=not args.very_quiet, check_digests=not args.no_check_digests): num_good += 1 + elif path_type == 'file': + log.info("Validating separate OCFL Inventory at " + path) + obj = ocfl.Object(lax_digests=args.lax_digests) + if obj.validate_inventory(path, + show_warnings=show_warnings, + show_errors=not args.very_quiet): + num_good += 1 else: log.error("Bad path %s (%s)" % (path, path_type)) if num_paths > 1: diff --git a/ocfl/_version.py b/ocfl/_version.py index de3568d6..72f5f253 100644 --- a/ocfl/_version.py +++ b/ocfl/_version.py @@ -1,2 +1,2 @@ """Version number for this Python implementation of OCFL.""" -__version__ = '1.1.1' +__version__ = '1.2.0' diff --git a/ocfl/object.py b/ocfl/object.py index a697db0a..bced2741 100755 --- a/ocfl/object.py +++ b/ocfl/object.py @@ -2,6 +2,7 @@ """Core of OCFL Object library.""" import copy import fs +import fs.path import fs.copy import hashlib import json @@ -19,7 +20,7 @@ from .object_utils import remove_first_directory, make_unused_filepath, next_version from .pyfs import open_fs from .namaste import Namaste -from .validator import Validator +from .validator import Validator, ValidatorAbortException from .version_metadata import VersionMetadata INVENTORY_FILENAME = 'inventory.json' @@ -540,6 +541,28 @@ def validate(self, objdir, show_warnings=True, show_errors=True, check_digests=T self.log.info("OCFL object at %s is INVALID" % (objdir)) return passed + def validate_inventory(self, path, show_warnings=True, show_errors=True): + """Validate just an OCFL Object inventory at path.""" + validator = Validator(show_warnings=show_warnings, + show_errors=show_errors) + try: + (inv_dir, inv_file) = fs.path.split(path) + validator.obj_fs = open_fs(inv_dir, create=False) + validator.validate_inventory(inv_file, where='standalone') + except fs.errors.ResourceNotFound: + validator.log.error('E033', where='standalone', explanation='failed to open directory') + except ValidatorAbortException: + pass + passed = (validator.log.num_errors == 0) + messages = str(validator) + if messages != '': + print(messages) + if passed: + self.log.info("Standalone OCFL inventory at %s is VALID" % (path)) + else: + self.log.info("Standalone OCFL inventory at %s is INVALID" % (path)) + return passed + def extract(self, objdir, version, dstdir): """Extract version from object at objdir into dstdir. diff --git a/ocfl/object_utils.py b/ocfl/object_utils.py index 332ac580..6ad25355 100755 --- a/ocfl/object_utils.py +++ b/ocfl/object_utils.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- """Utility functions to support the OCFL Object library.""" import fs +import fs.path import os -import os.path import re import sys try: @@ -50,7 +50,7 @@ def add_shared_args(parser): def check_shared_args(args): """Check arguments set with add_shared_args.""" if args.version: - print("%s is part of ocfl-py version %s" % (os.path.basename(sys.argv[0]), __version__)) + print("%s is part of ocfl-py version %s" % (fs.path.basename(sys.argv[0]), __version__)) sys.exit(0) @@ -85,12 +85,12 @@ def remove_first_directory(path): # split and rejoins, excluding the first directory rpath = '' while True: - (head, tail) = os.path.split(path) + (head, tail) = fs.path.split(path) if head == path or tail == path: break else: path = head - rpath = tail if rpath == '' else os.path.join(tail, rpath) + rpath = tail if rpath == '' else fs.path.join(tail, rpath) return rpath @@ -105,15 +105,34 @@ def make_unused_filepath(filepath, used, separator='__'): def find_path_type(path): - """Return a string indicating the type of thing, object or root, at path. + """Return a string indicating the type of thing at the given path. - Looks only at "0=*" Namaste files to determing the path type. Will return - an error description if not an `object` or storage `root`. + Return values: + 'root' - looks like an OCFL Storage Root + 'object' - looks like an OCFL Object + 'file' - a file, might be an inventory + other string explains error description + + Looks only at "0=*" Namaste files to determine the directory type. """ try: pyfs = open_fs(path, create=False) except (fs.opener.errors.OpenerError, fs.errors.CreateFailed) as e: - return("does not exist or cannot be opened (" + str(e) + ")") + # Failed to open path as a filesystem, try enclosing directory + # in case path is a file + (parent, filename) = fs.path.split(path) + try: + pyfs = open_fs(parent, create=False) + except (fs.opener.errors.OpenerError, fs.errors.CreateFailed) as e: + return("path cannot be opened, and nor can parent (" + str(e) + ")") + # Can open parent, is filename a file there? + try: + info = pyfs.getinfo(filename) + except fs.errors.ResourceNotFound: + return("path does not exist") + if info.is_dir: + return("directory that could not be opened as a filesystem, this should not happen") # pragma: no cover + return('file') namastes = find_namastes(0, pyfs=pyfs) if len(namastes) == 0: return("no 0= declaration file") diff --git a/tests/test_namaste.py b/tests/test_namaste.py index c41951ec..3fb922e2 100644 --- a/tests/test_namaste.py +++ b/tests/test_namaste.py @@ -91,6 +91,14 @@ def test15_check_content(self): self.assertRaises(NamasteException, Namaste().check_content, 'tests/testdata/namaste') self.assertRaises(NamasteException, Namaste(0, 'a').check_content, 'tests/testdata/namaste/does_not_exist') self.assertRaises(NamasteException, Namaste(0, 'bison').check_content, 'tests/testdata/namaste') + # Using pyfs... + tmpfs = fs.tempfs.TempFS() + tmpfs.writetext('9=niner', 'niner\n') + tmpfs.writetext('8=smiley', 'FROWNY\n') + Namaste(9, 'niner').check_content(pyfs=tmpfs) + Namaste(9, 'niner').check_content(dir='', pyfs=tmpfs) + self.assertRaises(NamasteException, Namaste(8, 'smiley').check_content, pyfs=tmpfs) + self.assertRaises(NamasteException, Namaste(7, 'not-there').check_content, pyfs=tmpfs) def test16_content_ok(self): """Test content_ok method.""" diff --git a/tests/test_object.py b/tests/test_object.py index e319c167..6a64a424 100644 --- a/tests/test_object.py +++ b/tests/test_object.py @@ -21,7 +21,7 @@ def assertRegex(self, *args, **kwargs): """Hack for Python 2.7.""" return self.assertRegexpMatches(*args, **kwargs) - def test01_init(self): + def test00_init(self): """Test Object init.""" oo = Object() self.assertEqual(oo.id, None) @@ -33,6 +33,14 @@ def test01_init(self): self.assertEqual(oo.digest_algorithm, 'sha1') self.assertEqual(oo.fixity, ['md5', 'crc16']) + def test01_open_fs(self): + """Test open_fs.""" + oo = Object() + self.assertEqual(oo.obj_fs, None) + oo.open_fs('tests') + self.assertNotEqual(oo.obj_fs, None) + self.assertRaises(ObjectException, oo.open_fs, 'tests/testdata/i_do_not_exist') + def test02_parse_version_directory(self): """Test parse_version_directory.""" oo = Object() @@ -304,7 +312,7 @@ def test12_show(self): self.assertTrue('├── 0=ocfl_object_1.0' in out) # FIXME - need real tests in here when there is real output - def test13_validate(self): + def test_validate(self): """Test validate method.""" oo = Object() self.assertTrue(oo.validate(objdir='fixtures/1.0/good-objects/minimal_one_version_one_file')) @@ -313,7 +321,16 @@ def test13_validate(self): self.assertFalse(oo.validate(objdir='fixtures/1.0/bad-objects/E001_no_decl')) self.assertFalse(oo.validate(objdir='fixtures/1.0/bad-objects/E036_no_id')) - def test14_parse_inventory(self): + def test_validate_inventory(self): + """Test validate_inventory method.""" + oo = Object() + self.assertTrue(oo.validate_inventory(path='fixtures/1.0/good-objects/minimal_one_version_one_file/inventory.json')) + # Error cases + self.assertFalse(oo.validate_inventory(path='tests/testdata/i_do_not_exist')) + self.assertFalse(oo.validate_inventory(path='fixtures/1.0/bad-objects/E036_no_id/inventory.json')) + self.assertFalse(oo.validate_inventory(path='tests/testdata//namaste/0=frog')) # not JSON + + def test_parse_inventory(self): """Test parse_inventory method.""" oo = Object() oo.open_fs('fixtures/1.0/good-objects/minimal_one_version_one_file') @@ -336,7 +353,7 @@ def test14_parse_inventory(self): oo.open_fs('fixtures/1.0/bad-objects/E036_no_id') self.assertRaises(ObjectException, oo.parse_inventory) - def test15_map_filepath(self): + def test_map_filepath(self): """Test map_filepath method.""" oo = Object() # default is uri @@ -353,7 +370,7 @@ def test15_map_filepath(self): oo.filepath_normalization = '???' self.assertRaises(Exception, oo.map_filepath, 'a', 'v1', {}) - def test16_extract(self): + def test_extract(self): """Test extract method.""" tempdir = tempfile.mkdtemp(prefix='test_extract') oo = Object() diff --git a/tests/test_object_utils.py b/tests/test_object_utils.py index 3b4162a5..826107ae 100644 --- a/tests/test_object_utils.py +++ b/tests/test_object_utils.py @@ -2,7 +2,7 @@ """Object tests.""" import argparse import unittest -from ocfl.object_utils import remove_first_directory, make_unused_filepath, next_version, add_object_args, find_path_type +from ocfl.object_utils import remove_first_directory, make_unused_filepath, next_version, add_object_args, add_shared_args, check_shared_args, find_path_type class TestAll(unittest.TestCase): @@ -48,12 +48,29 @@ def test_add_object_args(self): args = parser.parse_args(['--skip', 'aa']) self.assertIn('aa', args.skip) + def test_add_shared_args(self): + """Test (kinda) adding shared args.""" + parser = argparse.ArgumentParser() + add_shared_args(parser) + args = parser.parse_args(['--version', '-v']) + self.assertTrue(args.version) + self.assertTrue(args.verbose) + + def test_check_shared_args(self): + """Test check of shared args.""" + parser = argparse.ArgumentParser() + add_shared_args(parser) + args = parser.parse_args(['--version', '-v']) + check_shared_args(parser.parse_args(['-v'])) + self.assertRaises(SystemExit, check_shared_args, parser.parse_args(['--version'])) + def test_find_path_type(self): """Test find_path_type function.""" + self.assertEqual(find_path_type("extra_fixtures/good-storage-roots/fedora-root"), "root") + self.assertEqual(find_path_type("fixtures/1.0/good-objects/minimal_one_version_one_file"), "object") + self.assertEqual(find_path_type("README"), "file") self.assertIn("does not exist", find_path_type("this_path_does_not_exist")) - self.assertIn("cannot be opened", find_path_type("ocfl-object.py")) + self.assertIn("nor can parent", find_path_type("still_nope/nope_doesnt_exist")) self.assertEqual(find_path_type("ocfl"), "no 0= declaration file") self.assertIn("more than one 0= declaration file", find_path_type("extra_fixtures/misc/multiple_declarations")) self.assertIn("unrecognized", find_path_type("extra_fixtures/misc/unknown_declaration")) - self.assertEqual(find_path_type("extra_fixtures/good-storage-roots/fedora-root"), "root") - self.assertEqual(find_path_type("fixtures/1.0/good-objects/minimal_one_version_one_file"), "object")