Merge pull request #219 from stac-utils/item_collection

Validate item collections

Author: jonhealy1

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 repos:
--   repo: https://gitlab.com/pycqa/flake8
+-   repo: https://github.com/PyCQA/flake8
     rev: 3.9.1
     hooks:
     - id: flake8

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 
 The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v3.3.0] - 2022-11-28
+
+### Added
+
+- Added --item-collection to validate local and remote item collections https://github.com/stac-utils/stac-validator/pull/219
+- Added --pages to validate additional items retrieved via pagination links https://github.com/stac-utils/stac-validator/pull/219
+
 ## [v3.2.0] - 2022-09-20
 
 ### Added  
@@ -164,7 +171,11 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/)
 - With the newest version - 1.0.0-beta.2 - items will run through jsonchema validation before the PySTAC validation. The reason for this is that jsonschema will give more informative error messages. This should be addressed better in the future. This is not the case with the --recursive option as time can be a concern here with larger collections.
 - Logging. Various additions were made here depending on the options selected. This was done to help assist people to update their STAC collections.
 
-[v3.0.0]: <https://github.com/sparkgeo/stac-validator/compare/v2.5.0..main>
+[Unreleased]: <https://github.com/sparkgeo/stac-validator/compare/v3.3.0..main>
+[v3.3.0]: <https://github.com/sparkgeo/stac-validator/compare/v3.2.0..v3.3.0>
+[v3.2.0]: <https://github.com/sparkgeo/stac-validator/compare/v3.1.0..v3.2.0>
+[v3.1.0]: <https://github.com/sparkgeo/stac-validator/compare/v3.0.0..v3.1.0>
+[v3.0.0]: <https://github.com/sparkgeo/stac-validator/compare/v2.5.0..v3.0.0>
 [v2.5.0]: <https://github.com/sparkgeo/stac-validator/compare/v2.4.3..v2.5.0>
 [v2.4.3]: <https://github.com/sparkgeo/stac-validator/compare/v2.3.0..v2.4.0>
 [v2.4.2]: <https://github.com/sparkgeo/stac-validator/compare/v2.4.1..v2.4.2>

README.md

@@ -102,6 +102,10 @@ Options:
   -m, --max-depth INTEGER  Maximum depth to traverse when recursing. Omit this
                            argument to get full recursion. Ignored if
                            `recursive == False`.
+  --item-collection        Validate item collection response. Can be combined
+                           with --pages. Defaults to one page.
+  -p, --pages INTEGER      Maximum number of pages to validate via --item-
+                           collection. Defaults to one page.
   -v, --verbose            Enables verbose output for recursive mode.
   --no_output              Do not print output to console.
   --log_file TEXT          Save full recursive output to log file (local
@@ -200,6 +204,16 @@ stac = stac_validator.StacValidate()
 stac.validate_dict(dictionary)
 print(stac.message)
 ```
+
+**Item Collection**
+  
+```python
+from stac_validator import stac_validator
+  
+stac = stac_validator.StacValidate()
+stac.validate_item_collection_dict(item_collection_dict)
+print(stac.message)
+```
 ---
 
 # Testing
@@ -305,3 +319,8 @@ stac-validator https://spot-canada-ortho.s3.amazonaws.com/catalog.json --recursi
     }
 ]
 ```
+**--item-collection**
+
+```bash
+stac-validator https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a/items --item_collection --pages 2
+```

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup
 
-__version__ = "3.2.0"
+__version__ = "3.3.0"
 
 with open("README.md", "r") as fh:
     long_description = fh.read()

stac_validator/stac_validator.py

@@ -18,6 +18,17 @@ def print_update_message(version):
     click.secho()
 
 
+def item_collection_summary(message):
+    valid_count = 0
+    for item in message:
+        if "valid_stac" in item and item["valid_stac"] is True:
+            valid_count = valid_count + 1
+    click.secho()
+    click.secho("--item-collection summary", bold=True)
+    click.secho(f"items_validated: {len(message)}")
+    click.secho(f"valid_items: {valid_count}")
+
+
 @click.command()
 @click.argument("stac_file")
 @click.option(
@@ -52,6 +63,17 @@ def print_update_message(version):
     type=int,
     help="Maximum depth to traverse when recursing. Omit this argument to get full recursion. Ignored if `recursive == False`.",
 )
+@click.option(
+    "--item-collection",
+    is_flag=True,
+    help="Validate item collection response. Can be combined with --pages. Defaults to one page.",
+)
+@click.option(
+    "--pages",
+    "-p",
+    type=int,
+    help="Maximum number of pages to validate via --item-collection. Defaults to one page.",
+)
 @click.option(
     "-v", "--verbose", is_flag=True, help="Enables verbose output for recursive mode."
 )
@@ -64,6 +86,8 @@ def print_update_message(version):
 @click.version_option(version=pkg_resources.require("stac-validator")[0].version)
 def main(
     stac_file,
+    item_collection,
+    pages,
     recursive,
     max_depth,
     core,
@@ -79,6 +103,8 @@ def main(
     valid = True
     stac = StacValidate(
         stac_file=stac_file,
+        item_collection=item_collection,
+        pages=pages,
         recursive=recursive,
         max_depth=max_depth,
         core=core,
@@ -90,7 +116,10 @@ def main(
         no_output=no_output,
         log=log_file,
     )
-    valid = stac.run()
+    if not item_collection:
+        valid = stac.run()
+    else:
+        stac.validate_item_collection()
 
     message = stac.message
     if "version" in message[0]:
@@ -99,6 +128,9 @@ def main(
     if no_output is False:
         click.echo(json.dumps(message, indent=4))
 
+    if item_collection:
+        item_collection_summary(message)
+
     sys.exit(0 if valid else 1)
 
 

stac_validator/validate.py

@@ -1,7 +1,7 @@
 import json
 import os
 from json.decoder import JSONDecodeError
-from typing import List, Optional
+from typing import Optional
 from urllib.error import HTTPError, URLError
 
 import click  # type: ignore
@@ -22,7 +22,9 @@
 class StacValidate:
     def __init__(
         self,
-        stac_file: str = None,
+        stac_file: Optional[str] = None,
+        item_collection: bool = False,
+        pages: Optional[int] = None,
         recursive: bool = False,
         max_depth: Optional[int] = None,
         core: bool = False,
@@ -35,6 +37,8 @@ def __init__(
         log: str = "",
     ):
         self.stac_file = stac_file
+        self.item_collection = item_collection
+        self.pages = pages
         self.message: list = []
         self.custom = custom
         self.links = links
@@ -64,10 +68,10 @@ def create_err_msg(self, err_type: str, err_msg: str) -> dict:
         }
 
     def create_links_message(self):
-        format_valid: List[str] = []
-        format_invalid: List[str] = []
-        request_valid: List[str] = []
-        request_invalid: List[str] = []
+        format_valid = []
+        format_invalid = []
+        request_valid = []
+        request_invalid = []
         return {
             "format_valid": format_valid,
             "format_invalid": format_invalid,
@@ -267,75 +271,106 @@ def recursive_validator(self, stac_type: str) -> bool:
                         click.echo(json.dumps(message, indent=4))
         return True
 
-    def validate_dict(cls, stac_content):
-        cls.stac_content = stac_content
-        return cls.run()
+    def validate_dict(self, stac_content):
+        self.stac_content = stac_content
+        return self.run()
 
-    def run(cls):
+    def validate_item_collection_dict(self, item_collection):
+        for item in item_collection["features"]:
+            self.custom = ""
+            self.validate_dict(item)
+
+    def validate_item_collection(self):
+        page = 1
+        print(f"processing page {page}")
+        item_collection = fetch_and_parse_file(self.stac_file)
+        self.validate_item_collection_dict(item_collection)
+        try:
+            if self.pages is not None:
+                for _ in range(self.pages - 1):
+                    if "links" in item_collection:
+                        for link in item_collection["links"]:
+                            if link["rel"] == "next":
+                                page = page + 1
+                                print(f"processing page {page}")
+                                next_link = link["href"]
+                                self.stac_file = next_link
+                                item_collection = fetch_and_parse_file(self.stac_file)
+                                self.validate_item_collection_dict(item_collection)
+                                break
+        except Exception as e:
+            message = {}
+            message["pagination_error"] = (
+                f"Validating the item collection failed on page {page}: ",
+                str(e),
+            )
+            self.message.append(message)
+
+    def run(self):
         message = {}
         try:
-            if cls.stac_file is not None:
-                cls.stac_content = fetch_and_parse_file(cls.stac_file)
-            stac_type = get_stac_type(cls.stac_content).upper()
-            cls.version = cls.stac_content["stac_version"]
+            if self.stac_file is not None and self.item_collection is False:
+                self.stac_content = fetch_and_parse_file(self.stac_file)
+            stac_type = get_stac_type(self.stac_content).upper()
+            self.version = self.stac_content["stac_version"]
 
-            if cls.core is True:
-                message = cls.create_message(stac_type, "core")
-                cls.core_validator(stac_type)
-                message["schema"] = [cls.custom]
-                cls.valid = True
-            elif cls.custom != "":
-                message = cls.create_message(stac_type, "custom")
-                message["schema"] = [cls.custom]
-                cls.custom_validator()
-                cls.valid = True
-            elif cls.recursive:
-                cls.valid = cls.recursive_validator(stac_type)
-            elif cls.extensions is True:
-                message = cls.extensions_validator(stac_type)
+            if self.core is True:
+                message = self.create_message(stac_type, "core")
+                self.core_validator(stac_type)
+                message["schema"] = [self.custom]
+                self.valid = True
+            elif self.custom != "":
+                message = self.create_message(stac_type, "custom")
+                message["schema"] = [self.custom]
+                self.custom_validator()
+                self.valid = True
+            elif self.recursive:
+                self.valid = self.recursive_validator(stac_type)
+            elif self.extensions is True:
+                message = self.extensions_validator(stac_type)
             else:
-                cls.valid = True
-                message = cls.default_validator(stac_type)
+                self.valid = True
+                message = self.default_validator(stac_type)
 
         except URLError as e:
-            message.update(cls.create_err_msg("URLError", str(e)))
+            message.update(self.create_err_msg("URLError", str(e)))
         except JSONDecodeError as e:
-            message.update(cls.create_err_msg("JSONDecodeError", str(e)))
+            message.update(self.create_err_msg("JSONDecodeError", str(e)))
         except ValueError as e:
-            message.update(cls.create_err_msg("ValueError", str(e)))
+            message.update(self.create_err_msg("ValueError", str(e)))
         except TypeError as e:
-            message.update(cls.create_err_msg("TypeError", str(e)))
+            message.update(self.create_err_msg("TypeError", str(e)))
         except FileNotFoundError as e:
-            message.update(cls.create_err_msg("FileNotFoundError", str(e)))
+            message.update(self.create_err_msg("FileNotFoundError", str(e)))
         except ConnectionError as e:
-            message.update(cls.create_err_msg("ConnectionError", str(e)))
+            message.update(self.create_err_msg("ConnectionError", str(e)))
         except exceptions.SSLError as e:
-            message.update(cls.create_err_msg("SSLError", str(e)))
+            message.update(self.create_err_msg("SSLError", str(e)))
         except OSError as e:
-            message.update(cls.create_err_msg("OSError", str(e)))
+            message.update(self.create_err_msg("OSError", str(e)))
         except jsonschema.exceptions.ValidationError as e:
             if e.absolute_path:
                 err_msg = f"{e.message}. Error is in {' -> '.join([str(i) for i in e.absolute_path])} "
             else:
                 err_msg = f"{e.message} of the root of the STAC object"
-            message.update(cls.create_err_msg("JSONSchemaValidationError", err_msg))
+            message.update(self.create_err_msg("JSONSchemaValidationError", err_msg))
         except KeyError as e:
-            message.update(cls.create_err_msg("KeyError", str(e)))
+            message.update(self.create_err_msg("KeyError", str(e)))
         except HTTPError as e:
-            message.update(cls.create_err_msg("HTTPError", str(e)))
+            message.update(self.create_err_msg("HTTPError", str(e)))
         except Exception as e:
-            message.update(cls.create_err_msg("Exception", str(e)))
+            message.update(self.create_err_msg("Exception", str(e)))
 
         if len(message) > 0:
-            message["valid_stac"] = cls.valid
-            cls.message.append(message)
+            message["valid_stac"] = self.valid
+            self.message.append(message)
 
-        if cls.log != "":
-            f = open(cls.log, "w")
-            f.write(json.dumps(cls.message, indent=4))
+        if self.log != "":
+            f = open(self.log, "w")
+            f.write(json.dumps(self.message, indent=4))
             f.close()
 
-        if cls.valid:
+        if self.valid:
             return True
         else:
             return False

tests/test_assets.py

@@ -2,7 +2,6 @@
 Description: Test --links option
 
 """
-__authors__ = "James Banting", "Jonathan Healy"
 
 import json
 

tests/test_core.py

@@ -2,7 +2,6 @@
 Description: Test the validator for core functionality without validating extensions
 
 """
-__authors__ = "James Banting", "Jonathan Healy"
 
 from stac_validator import stac_validator
 

tests/test_custom.py

@@ -2,7 +2,6 @@
 Description: Test the custom option for custom schemas
 
 """
-__authors__ = "James Banting", "Jonathan Healy"
 
 from stac_validator import stac_validator